diff --git a/.cursor/rules/formatting.mdc b/.cursor/rules/formatting.mdc deleted file mode 100644 index 1c91aa898..000000000 --- a/.cursor/rules/formatting.mdc +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: formatting -description: Checks for code formatting issues. ---- - -# Overview - -Before pushing any Rust code, make sure to run `cargo +nightly fmt` to catch formatting issues. -Before pushing any Python code, make sure to run `hatch run fmt` to catch formatting issues. -Before pushing any documentation changes, make sure to run `pnpm run format` to catch formatting issues. diff --git a/.cursor/rules/python-env.mdc b/.cursor/rules/python-env.mdc deleted file mode 100644 index 38ae60ecb..000000000 --- a/.cursor/rules/python-env.mdc +++ /dev/null @@ -1,4 +0,0 @@ ---- -alwaysApply: true ---- -Make sure to use Hatch for Python for this project. diff --git a/.github/agents/spark-test-expert.agent.md b/.github/agents/spark-test-expert.agent.md index 516362a69..d39c8e5fb 100644 --- a/.github/agents/spark-test-expert.agent.md +++ b/.github/agents/spark-test-expert.agent.md @@ -12,8 +12,8 @@ Once you have identified the failed test cases from the logs, or if the failed t Run the tests inside the Hatch environments defined in `pyproject.toml`. The following test environments are currently available: -- `test-spark.spark-3.5.7` -- `test-spark.spark-4.1.1` +- `test-spark.spark-3.5.9` +- `test-spark.spark-4.2.0` - `test-ibis` The `test-spark.spark-*` environments support running PySpark unit tests and doctests. The `test-ibis` environment supports running Ibis unit tests with PySpark as the backend. diff --git a/.github/instructions/dev.instructions.md b/.github/instructions/dev.instructions.md index 6b77c72cc..2d2ddbe83 100644 --- a/.github/instructions/dev.instructions.md +++ b/.github/instructions/dev.instructions.md @@ -20,6 +20,8 @@ Tests use "gold data" files for validation. If you change logic that affects out env SAIL_UPDATE_GOLD_DATA=1 cargo nextest run ``` +Please do not add test cases manually to the gold data files. + ### Python Development - **Formatting**: `hatch fmt` diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eefc20a5a..51d2ec1cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,7 +108,7 @@ jobs: strategy: fail-fast: false matrix: - spark-version: ["3.5.7", "4.1.1"] + spark-version: ["3.5.9", "4.2.0"] with: spark_version: ${{ matrix.spark-version }} needs: @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - spark-version: ["3.5.7", "4.0.1", "4.1.1"] + spark-version: ["3.5.9", "4.0.4", "4.1.3", "4.2.0"] with: spark_version: ${{ matrix.spark-version }} needs: diff --git a/.github/workflows/catalog-tests.yml b/.github/workflows/catalog-tests.yml index 9007f1a16..b1185e99c 100644 --- a/.github/workflows/catalog-tests.yml +++ b/.github/workflows/catalog-tests.yml @@ -11,7 +11,7 @@ jobs: name: Test runs-on: ubuntu-latest env: - HATCH_ENV: "test.spark-4.1.1" + HATCH_ENV: "test.spark-4.2.0" # Coverage environment variables RUSTC_WORKSPACE_WRAPPER: ${{ github.workspace }}/.github/scripts/rustc-workspace-wrapper.sh LLVM_PROFILE_FILE: ${{ github.workspace }}/target/sail-catalog-it-%p-%m.profraw @@ -75,5 +75,5 @@ jobs: files: coverage-catalog-it.info flags: catalog-integration-tests name: catalog-integration-tests - fail_ci_if_error: true + fail_ci_if_error: ${{ github.repository == 'lakehq/sail' }} env_vars: OS diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 32e36508b..1f2749276 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -21,6 +21,7 @@ on: jobs: docs-deploy: name: Docs Deploy + if: ${{ github.repository == 'lakehq/sail' }} runs-on: ubuntu-slim environment: docs/${{ inputs.stage }} permissions: diff --git a/.github/workflows/gold-data-script-validation.yml b/.github/workflows/gold-data-script-validation.yml index da3507731..a10471cc9 100644 --- a/.github/workflows/gold-data-script-validation.yml +++ b/.github/workflows/gold-data-script-validation.yml @@ -26,7 +26,7 @@ jobs: with: repository: apache/spark path: opt/spark - ref: v4.1.1 + ref: v4.2.0 fetch-depth: 1 - uses: actions/setup-java@v5 diff --git a/.github/workflows/ibis-tests.yml b/.github/workflows/ibis-tests.yml index 144b01a3f..09237a648 100644 --- a/.github/workflows/ibis-tests.yml +++ b/.github/workflows/ibis-tests.yml @@ -110,7 +110,7 @@ jobs: files: coverage-ibis.info flags: ibis-tests name: ibis-tests - fail_ci_if_error: true + fail_ci_if_error: ${{ github.repository == 'lakehq/sail' }} env_vars: OS - name: Upload Test Logs diff --git a/.github/workflows/multi-platform-build.yml b/.github/workflows/multi-platform-build.yml index d7f3524e9..96253e47f 100644 --- a/.github/workflows/multi-platform-build.yml +++ b/.github/workflows/multi-platform-build.yml @@ -65,13 +65,13 @@ jobs: platforms.push({ runner: 'ubuntu-24.04-arm', target: 'aarch64', os: 'linux' }); } if (context.payload.inputs.build_macos_x86_64 == 'true') { - platforms.push({ runner: 'macos-15-intel', target: 'x86_64', os: 'macos' }); + platforms.push({ runner: 'macos-26-intel', target: 'x86_64', os: 'macos' }); } if (context.payload.inputs.build_macos_aarch64 == 'true') { - platforms.push({ runner: 'macos-14', target: 'aarch64', os: 'macos' }); + platforms.push({ runner: 'macos-26', target: 'aarch64', os: 'macos' }); } if (context.payload.inputs.build_windows_x64 == 'true') { - platforms.push({ runner: 'windows-2022', target: 'x64', os: 'windows' }); + platforms.push({ runner: 'windows-2025', target: 'x64', os: 'windows' }); } return platforms; diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index d3ba0596c..0d341106e 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -90,5 +90,5 @@ jobs: files: coverage-python-${{ inputs.spark_version }}.info flags: python-unit-tests name: python-unit-tests - fail_ci_if_error: true + fail_ci_if_error: ${{ github.repository == 'lakehq/sail' }} env_vars: OS diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c26bf6cd..74c18d7ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -149,13 +149,13 @@ jobs: - runner: ubuntu-24.04-x64-8-core target: aarch64 os: linux - - runner: macos-15-intel + - runner: macos-26-intel target: x86_64 os: macos - - runner: macos-14 + - runner: macos-26 target: aarch64 os: macos - - runner: windows-2022 + - runner: windows-2025 target: x64 os: windows steps: @@ -251,9 +251,9 @@ jobs: platform: - runner: ubuntu-24.04-x64-8-core - runner: ubuntu-24.04-arm-8-core - - runner: macos-15-intel - - runner: macos-14 - - runner: windows-2022 + - runner: macos-26-intel + - runner: macos-26 + - runner: windows-2025 needs: - setup - build-sdist @@ -312,9 +312,9 @@ jobs: platform: - runner: ubuntu-24.04 - runner: ubuntu-24.04-arm - - runner: macos-15-intel - - runner: macos-14 - - runner: windows-2022 + - runner: macos-26-intel + - runner: macos-26 + - runner: windows-2025 needs: - setup - build-wheel @@ -351,19 +351,21 @@ jobs: - test-sdist - test-wheel steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v7 with: pattern: "package-*" path: dist merge-multiple: true + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Show Release Artifacts shell: bash - run: | - find dist -name '*.tar.gz' -exec echo '{}' \; -exec tar -tvf '{}' \; - find dist -name '*.whl' -exec unzip -l '{}' \; - ls -lah dist - md5sum dist/* + run: scripts/release/review_packages.py --path dist >> "$GITHUB_STEP_SUMMARY" release-pypi: name: Release to PyPI diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 4d6f04418..ad7430446 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -27,16 +27,16 @@ jobs: continue-on-error: true with: workflow_run_id: ${{ github.event.workflow_run.id }} - artifact_name: spark-3.5.7-test-report - report_name: spark-3.5.7-test + artifact_name: spark-3.5.9-test-report + report_name: spark-3.5.9-test - uses: ./.github/actions/pull-request-report if: github.event.workflow_run.name == 'Build' continue-on-error: true with: workflow_run_id: ${{ github.event.workflow_run.id }} - artifact_name: spark-4.1.1-test-report - report_name: spark-4.1.1-test + artifact_name: spark-4.2.0-test-report + report_name: spark-4.2.0-test - uses: ./.github/actions/pull-request-report if: github.event.workflow_run.name == 'Build' diff --git a/.github/workflows/rust-tests.yml b/.github/workflows/rust-tests.yml index 114dc91ce..af94daa12 100644 --- a/.github/workflows/rust-tests.yml +++ b/.github/workflows/rust-tests.yml @@ -64,7 +64,7 @@ jobs: files: coverage-rust-unit.info flags: rust-unit-tests name: rust-unit-tests - fail_ci_if_error: true + fail_ci_if_error: ${{ github.repository == 'lakehq/sail' }} env_vars: OS - name: Install Kerberos client tools @@ -99,5 +99,5 @@ jobs: files: coverage-rust-slow.info flags: rust-slow-tests name: rust-slow-tests - fail_ci_if_error: true + fail_ci_if_error: ${{ github.repository == 'lakehq/sail' }} env_vars: OS diff --git a/.github/workflows/spark-package-artifacts.yml b/.github/workflows/spark-package-artifacts.yml index cec6c9041..b24273471 100644 --- a/.github/workflows/spark-package-artifacts.yml +++ b/.github/workflows/spark-package-artifacts.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - spark-version: ["3.5.7", "4.1.1"] + spark-version: ["3.5.9", "4.2.0"] steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/spark-tests.yml b/.github/workflows/spark-tests.yml index 06eebefdf..f0c0b6456 100644 --- a/.github/workflows/spark-tests.yml +++ b/.github/workflows/spark-tests.yml @@ -143,7 +143,7 @@ jobs: files: coverage-spark-${{ inputs.spark_version }}.info flags: spark-tests name: spark-tests - fail_ci_if_error: true + fail_ci_if_error: ${{ github.repository == 'lakehq/sail' }} env_vars: OS - name: Upload Test Logs diff --git a/.gitignore b/.gitignore index f4962f2c7..ee64fc07a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ out/ /opt/ # Editor directories and files -.cursor/ .idea/ .vscode/* *.iml @@ -100,11 +99,10 @@ docs/.vitepress/.temp # LLVM coverage files *.profraw -# Claude Code settings +# AI coding configuration files and artifacts +.cursor/ .claude/ .worktrees/ - CLAUDE.md AGENTS.md -# roborev snapshots -/.roborev/ +.roborev/ diff --git a/Cargo.lock b/Cargo.lock index 300db313c..f9715557a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -155,7 +155,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -184,7 +184,7 @@ dependencies = [ "serde_json", "strum", "strum_macros 0.27.2", - "thiserror 2.0.18", + "thiserror 2.0.19", "uuid", ] @@ -470,7 +470,7 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "serde", "serde_core", "serde_json", @@ -576,7 +576,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -598,18 +598,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -644,9 +644,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.18" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" dependencies = [ "aws-credential-types", "aws-runtime", @@ -675,9 +675,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -709,9 +709,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.5" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" +checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -734,9 +734,9 @@ dependencies = [ [[package]] name = "aws-sdk-glue" -version = "1.152.0" +version = "1.154.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267d3459c708985a1c1472bb2eb8def3b8673abbcdb3e8057d73e40fa5a7f5b4" +checksum = "3e53d85505932c48becf7ce44f1005a02feb8d981cec15542986b981d5ea964d" dependencies = [ "arc-swap", "aws-credential-types", @@ -747,6 +747,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -759,9 +760,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.102.0" +version = "1.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" +checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401" dependencies = [ "arc-swap", "aws-credential-types", @@ -772,6 +773,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -784,9 +786,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.104.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" +checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a" dependencies = [ "arc-swap", "aws-credential-types", @@ -797,6 +799,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -809,9 +812,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.107.0" +version = "1.108.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f" dependencies = [ "arc-swap", "aws-credential-types", @@ -823,6 +826,7 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -835,9 +839,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.5" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -857,9 +861,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -868,9 +872,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.63.6" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -889,9 +893,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.13" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -913,9 +917,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.7" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -924,18 +928,18 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77" dependencies = [ "aws-smithy-types", "urlencoding", @@ -943,9 +947,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.11.3" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" +checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -969,9 +973,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.12.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -987,20 +991,20 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "aws-smithy-schema" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -1009,9 +1013,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", @@ -1035,18 +1039,21 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.16" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1153,9 +1160,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "bitvec" @@ -1200,7 +1207,7 @@ checksum = "56791e4bd64c99fc361e01008f45c984baa93f12a0957d1b3c51dd2c6baab453" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1238,7 +1245,7 @@ checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" dependencies = [ "async-stream", "base64", - "bitflags 2.13.0", + "bitflags 2.13.1", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -1267,7 +1274,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -1329,7 +1336,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1353,7 +1360,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1437,7 +1444,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1448,9 +1455,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -1603,9 +1610,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" dependencies = [ "clap_builder", "clap_derive", @@ -1613,9 +1620,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" dependencies = [ "anstream", "anstyle", @@ -1625,14 +1632,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -1735,6 +1742,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-hex" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e2a781ebdf4467d1428dc4593067825fb646f6871475098d8577421af73558" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -1865,9 +1884,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1893,7 +1912,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "crossterm_winapi", "document-features", "parking_lot", @@ -1995,7 +2014,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2006,7 +2025,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2025,9 +2044,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", @@ -2080,9 +2099,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -2105,9 +2124,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -2128,9 +2147,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ "arrow", "arrow-ipc", @@ -2154,9 +2173,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -2165,9 +2184,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-compression", @@ -2201,9 +2220,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -2225,9 +2244,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" dependencies = [ "arrow", "arrow-avro", @@ -2244,9 +2263,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -2267,9 +2286,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2290,9 +2309,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ "arrow", "async-trait", @@ -2321,15 +2340,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", @@ -2349,9 +2368,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", "arrow-schema", @@ -2372,9 +2391,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", @@ -2384,9 +2403,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -2416,9 +2435,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ "arrow", "datafusion-common", @@ -2437,9 +2456,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ "arrow", "datafusion-common", @@ -2449,9 +2468,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2474,9 +2493,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", @@ -2490,9 +2509,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2507,9 +2526,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2517,20 +2536,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2548,9 +2567,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ "arrow", "datafusion-common", @@ -2570,9 +2589,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2585,9 +2604,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ "arrow", "chrono", @@ -2602,9 +2621,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2621,9 +2640,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ "arrow", "arrow-data", @@ -2654,9 +2673,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" dependencies = [ "arrow", "chrono", @@ -2681,9 +2700,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" dependencies = [ "arrow", "datafusion-common", @@ -2692,9 +2711,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2708,9 +2727,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2722,9 +2741,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" dependencies = [ "arrow", "bigdecimal", @@ -2752,9 +2771,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2820,7 +2839,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2873,7 +2892,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2884,7 +2903,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2946,14 +2965,14 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "encode_unicode" @@ -2987,7 +3006,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2999,14 +3018,14 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "env_filter" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -3014,9 +3033,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -3038,7 +3057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3084,24 +3103,24 @@ dependencies = [ [[package]] name = "fastrace" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74539685f6baa3c33251571272eac7939fc3b93dff97c30381d7a6267394e528" +checksum = "dcef89a7c5b37f7c13551af01f509cbd5d32a8644c27423623639eb923cc2973" dependencies = [ "fastant", "fastrace-macro", "parking_lot", "pin-project", - "rand 0.10.1", + "rand 0.10.2", "rtrb", "serde", ] [[package]] name = "fastrace-futures" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b6a02e77334be6072af69c93f49a07af3212de09b5cd95a9ee44be49ffb2ac" +checksum = "9dedfda50f2cf2fe2c7fb1840f42cc2f1cc2a82786f650f2f03da4684ac09faf" dependencies = [ "fastrace", "futures-core", @@ -3111,20 +3130,20 @@ dependencies = [ [[package]] name = "fastrace-macro" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390d4f754b751905fa9294716c4ead717ffb5fa733a0c2417358b7779fceaeed" +checksum = "2f1fdb97ada2fde7912c9cab29b560d550ccc467ca1f39c952ce4b05f8d152b3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] name = "fastrace-opentelemetry" -version = "0.16.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4644a8a7ce6d20d83e73d9562f323388e4d817470d40bcb51fa2fe328db58cd2" +checksum = "67d87b05a82ea59d6d2a11066f4e0cc1e169ce50bc8c276279249ba374936d91" dependencies = [ "fastrace", "log", @@ -3215,7 +3234,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "rustc_version", ] @@ -3280,9 +3299,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -3295,9 +3314,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -3305,15 +3324,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -3322,38 +3341,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -3375,7 +3394,7 @@ dependencies = [ "g2poly", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3454,11 +3473,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -3468,9 +3485,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -3489,7 +3508,7 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "libgit2-sys", "log", @@ -3498,9 +3517,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "h2" @@ -3561,17 +3580,6 @@ dependencies = [ "foldhash 0.1.5", ] -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -3600,7 +3608,7 @@ checksum = "51610510377a0847d53b78b53f9c6c9b7df3ffb300d1181b2e04f68bba363734" dependencies = [ "aes", "base64", - "bitflags 2.13.0", + "bitflags 2.13.1", "bumpalo", "bytes", "cbc", @@ -3626,7 +3634,7 @@ dependencies = [ "regex", "roxmltree", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "url", "uuid", @@ -3645,7 +3653,7 @@ dependencies = [ "futures", "hdfs-native", "object_store", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", ] @@ -3690,10 +3698,10 @@ dependencies = [ "log", "num_cpus", "rand 0.9.4", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "windows-sys 0.61.2", ] @@ -3727,12 +3735,9 @@ dependencies = [ [[package]] name = "html-escape" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" -dependencies = [ - "utf8-width", -] +checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" [[package]] name = "http" @@ -4402,7 +4407,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4518,8 +4523,8 @@ dependencies = [ "proc-macro2", "quote", "strum", - "syn 2.0.118", - "thiserror 2.0.18", + "syn 2.0.119", + "thiserror 2.0.19", ] [[package]] @@ -4600,7 +4605,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4645,7 +4650,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4698,7 +4703,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -4751,7 +4756,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tower", @@ -4773,7 +4778,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -4933,7 +4938,7 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "plain", "redox_syscall 0.8.1", @@ -5035,7 +5040,7 @@ dependencies = [ "quote", "regex-syntax 0.8.11", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5083,7 +5088,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.4.14", + "regex-automata 0.4.16", ] [[package]] @@ -5209,7 +5214,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5221,7 +5226,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5246,9 +5251,9 @@ dependencies = [ [[package]] name = "monostate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4cc965c89dd0615a9e822ff8002f7633d2466143d51bd58693e4b2c75aabad" +checksum = "fdc4dfdf312bb1b22fce127237589f4ee31652bc479f3d208332f32132aaccc9" dependencies = [ "monostate-impl", "serde", @@ -5257,13 +5262,13 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f5b99488110875b5904839d396c2cdfaf241ff6622638acb879cc7effad5de" +checksum = "a656449eeb444c77364a3033f013c51869eee320c0dff805eb53024c0dfd435b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -5286,7 +5291,7 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5312,7 +5317,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5354,7 +5359,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -5376,7 +5381,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5427,7 +5432,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5500,7 +5505,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5536,14 +5541,14 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.1", - "reqwest", + "rand 0.10.2", + "reqwest 0.12.28", "ring", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -5570,17 +5575,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "openapiv3" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8d427828b22ae1fff2833a03d8486c2c881367f1c336349f307f321e7f4d05" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_json", -] - [[package]] name = "openssl-probe" version = "0.2.1" @@ -5589,23 +5583,23 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", ] [[package]] name = "opentelemetry-appender-log" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e50c59a96bd6a723a4329c5db31eb04fa4488c5f141ae7b9d4fd587439e6ee1" +checksum = "d7b6863c2e4f280ace0924dce400d46b12c6ff76a720cf6d2c5847ca820e460d" dependencies = [ "log", "opentelemetry", @@ -5613,22 +5607,22 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", "http 1.4.2", "opentelemetry", - "reqwest", + "reqwest 0.13.4", ] [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http 1.4.2", "opentelemetry", @@ -5636,41 +5630,45 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest", - "thiserror 2.0.18", + "reqwest 0.13.4", + "serde_json", + "thiserror 2.0.19", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ + "base64", + "const-hex", "opentelemetry", "opentelemetry_sdk", "prost", + "serde", "tonic", "tonic-prost", ] [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", - "tokio-stream", ] [[package]] @@ -5849,7 +5847,7 @@ dependencies = [ "regex", "regex-syntax 0.8.11", "structmeta", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5921,7 +5919,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5970,7 +5968,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6047,7 +6045,7 @@ dependencies = [ "phf_shared 0.12.1", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6060,7 +6058,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6100,7 +6098,7 @@ dependencies = [ "protobuf", "serde", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", ] @@ -6137,7 +6135,7 @@ dependencies = [ "scoped-tls", "serde", "serde_yaml", - "syn 2.0.118", + "syn 2.0.119", "tempfile", "toml 0.9.12+spec-1.1.0", "tracing", @@ -6155,7 +6153,7 @@ dependencies = [ "bytes", "chumsky 0.10.1", "faststr", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6171,7 +6169,7 @@ dependencies = [ "faststr", "pilota", "pilota-thrift-parser", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6191,7 +6189,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6308,7 +6306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6354,9 +6352,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -6369,75 +6367,24 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "version_check", "yansi", ] [[package]] -name = "progenitor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2326f73d5326257514712436680ef8da4543ee47c0e9e0d501545c8909ee12e4" -dependencies = [ - "progenitor-client", - "progenitor-impl", - "progenitor-macro", -] - -[[package]] -name = "progenitor-client" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71a0beb939758f229cbae70a4889c7c76a4ac0e90f0b1e7ae9b4636a927d1018" -dependencies = [ - "bytes", - "futures-core", - "percent-encoding", - "reqwest", - "serde", - "serde_json", - "serde_urlencoded", -] - -[[package]] -name = "progenitor-impl" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f6d9109b04e005bbdec84cacec7e81cc15533f2b5dc505f0defc212d270c15" -dependencies = [ - "heck", - "http 1.4.2", - "indexmap 2.14.0", - "openapiv3", - "proc-macro2", - "quote", - "regex", - "schemars 0.8.22", - "serde", - "serde_json", - "syn 2.0.118", - "thiserror 2.0.18", - "typify", - "unicode-ident", -] - -[[package]] -name = "progenitor-macro" -version = "0.11.2" +name = "proptest" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46596c574831739c661f22923fe587399c61f5e3e79b73cc9a93644c72248d84" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ - "openapiv3", - "proc-macro2", - "progenitor-impl", - "quote", - "schemars 0.8.22", - "serde", - "serde_json", - "serde_tokenstream", - "serde_yaml", - "syn 2.0.118", + "bitflags 2.13.1", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax 0.8.11", + "unarray", ] [[package]] @@ -6467,7 +6414,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.118", + "syn 2.0.119", "tempfile", ] @@ -6481,7 +6428,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6576,7 +6523,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6585,7 +6532,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "memchr", "unicase", ] @@ -6642,7 +6589,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6654,7 +6601,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6687,7 +6634,7 @@ dependencies = [ "rustc-hash 2.1.2", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "web-time", @@ -6695,20 +6642,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.19", "tinyvec", "tracing", "web-time", @@ -6725,14 +6673,14 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -6788,9 +6736,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -6859,7 +6807,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" dependencies = [ "num-traits", - "rand 0.10.1", + "rand 0.10.2", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", ] [[package]] @@ -6893,13 +6859,13 @@ dependencies = [ [[package]] name = "readonly" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a62d85ed81ca5305dc544bd42c8804c5060b78ffa5ad3c64b0fb6a8c13d062" +checksum = "67a77cd6f1a55ff3cb1969c2618f73107efaa6de1daf796ad60ce8f737ae97c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -6919,7 +6885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6928,7 +6894,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -6937,7 +6903,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -6948,7 +6914,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6968,18 +6934,18 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.14", + "regex-automata 0.4.16", "regex-syntax 0.8.11", ] @@ -7005,9 +6971,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -7045,16 +7011,6 @@ dependencies = [ "icu_properties 1.5.1", ] -[[package]] -name = "regress" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" -dependencies = [ - "hashbrown 0.16.1", - "memchr", -] - [[package]] name = "rend" version = "0.4.2" @@ -7079,7 +7035,6 @@ dependencies = [ "base64", "bytes", "encoding_rs", - "futures-channel", "futures-core", "futures-util", "h2", @@ -7117,6 +7072,37 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -7186,7 +7172,7 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7261,11 +7247,11 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -7274,18 +7260,18 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", @@ -7354,7 +7340,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "sail-build-scripts" -version = "0.6.4" +version = "0.7.0" dependencies = [ "lazy_static", "prettyplease", @@ -7363,25 +7349,29 @@ dependencies = [ "regex", "serde", "serde_yaml", - "syn 2.0.118", + "syn 2.0.119", + "thiserror 2.0.19", ] [[package]] name = "sail-cache" -version = "0.6.4" +version = "0.7.0" dependencies = [ "chrono", "datafusion", + "datafusion-common", "log", "moka", "object_store", "sail-common", - "thiserror 2.0.18", + "sail-common-datafusion", + "sail-object-store", + "thiserror 2.0.19", ] [[package]] name = "sail-catalog" -version = "0.6.4" +version = "0.7.0" dependencies = [ "async-trait", "datafusion", @@ -7396,17 +7386,19 @@ dependencies = [ "serde", "serde_arrow", "serde_json", - "thiserror 2.0.18", + "tempfile", + "thiserror 2.0.19", "tokio", ] [[package]] name = "sail-catalog-glue" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "async-trait", "aws-config", + "aws-credential-types", "aws-sdk-glue", "aws-smithy-types", "sail-catalog", @@ -7417,11 +7409,11 @@ dependencies = [ [[package]] name = "sail-catalog-hms" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "async-trait", - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "datafusion", "futures", @@ -7441,12 +7433,14 @@ dependencies = [ [[package]] name = "sail-catalog-iceberg" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "async-trait", + "log", "percent-encoding", - "reqwest", + "reqwest 0.12.28", + "sail-build-scripts", "sail-catalog", "sail-common", "sail-common-datafusion", @@ -7454,7 +7448,9 @@ dependencies = [ "serde", "serde_json", "serde_with", + "tempfile", "testcontainers", + "thiserror 2.0.19", "tokio", "url", "uuid", @@ -7463,7 +7459,7 @@ dependencies = [ [[package]] name = "sail-catalog-memory" -version = "0.6.4" +version = "0.7.0" dependencies = [ "async-trait", "dashmap", @@ -7473,20 +7469,21 @@ dependencies = [ [[package]] name = "sail-catalog-onelake" -version = "0.6.4" +version = "0.7.0" dependencies = [ - "arrow", "async-trait", - "reqwest", + "object_store", "sail-catalog", + "sail-catalog-iceberg", + "sail-catalog-unity", "sail-common-datafusion", - "serde", "tokio", + "url", ] [[package]] name = "sail-catalog-system" -version = "0.6.4" +version = "0.7.0" dependencies = [ "async-trait", "datafusion", @@ -7500,32 +7497,30 @@ dependencies = [ [[package]] name = "sail-catalog-unity" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "async-trait", "chrono", - "prettyplease", - "progenitor", - "progenitor-client", - "reqwest", + "percent-encoding", + "reqwest 0.12.28", + "sail-build-scripts", "sail-catalog", "sail-common", "sail-common-datafusion", "secrecy", "serde", "serde_json", - "serde_yaml", - "syn 2.0.118", "tempfile", "testcontainers", + "thiserror 2.0.19", "tokio", "wiremock", ] [[package]] name = "sail-cli" -version = "0.6.4" +version = "0.7.0" dependencies = [ "clap", "log", @@ -7543,29 +7538,37 @@ dependencies = [ [[package]] name = "sail-common" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "arrow-buffer", "arrow-schema", + "axum", + "fastrace", "figment", "glob", "half", "iana-time-zone", + "log", "num-traits", "num_enum", + "pin-project-lite", "secrecy", "serde", "serde_json", "serde_yaml", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "toml 0.8.23", + "tonic", + "tonic-health", + "tonic-reflection", + "tower", ] [[package]] name = "sail-common-datafusion" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow-schema", "async-trait", @@ -7592,8 +7595,8 @@ dependencies = [ "serde_arrow", "serde_json", "serde_yaml", - "syn 2.0.118", - "thiserror 2.0.18", + "syn 2.0.119", + "thiserror 2.0.19", "tokio", "tonic", "tonic-prost-build", @@ -7601,7 +7604,7 @@ dependencies = [ [[package]] name = "sail-common-hms" -version = "0.6.4" +version = "0.7.0" dependencies = [ "pilota", "volo", @@ -7611,7 +7614,7 @@ dependencies = [ [[package]] name = "sail-data-source" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "arrow-schema", @@ -7643,7 +7646,7 @@ dependencies = [ "sail-pyarrow", "sail-sql-analyzer", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "url", @@ -7651,7 +7654,7 @@ dependencies = [ [[package]] name = "sail-delta-lake" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow-schema", "async-trait", @@ -7688,7 +7691,7 @@ dependencies = [ "serde", "serde_arrow", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "url", "uuid", @@ -7696,7 +7699,7 @@ dependencies = [ [[package]] name = "sail-execution" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow-flight", "chrono", @@ -7712,7 +7715,7 @@ dependencies = [ "object_store", "prost", "prost-build", - "rand 0.10.1", + "rand 0.10.2", "readonly", "sail-catalog", "sail-catalog-system", @@ -7723,13 +7726,13 @@ dependencies = [ "sail-function", "sail-iceberg", "sail-logical-plan", + "sail-object-store", "sail-physical-plan", "sail-python-udf", - "sail-server", "sail-telemetry", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tonic", "tonic-prost", @@ -7741,7 +7744,7 @@ dependencies = [ [[package]] name = "sail-flight" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "arrow-flight", @@ -7754,13 +7757,13 @@ dependencies = [ "sail-catalog-memory", "sail-common", "sail-common-datafusion", + "sail-execution", "sail-plan", - "sail-server", "sail-session", "sail-sql-analyzer", "sail-sql-parser", "sail-telemetry", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tonic", "tonic-build", @@ -7770,7 +7773,7 @@ dependencies = [ [[package]] name = "sail-function" -version = "0.6.4" +version = "0.7.0" dependencies = [ "aes", "aes-gcm", @@ -7799,7 +7802,7 @@ dependencies = [ "parquet-variant-compute", "parquet-variant-json", "percent-encoding", - "rand 0.10.1", + "rand 0.10.2", "rand_chacha 0.10.0", "rand_distr", "regex", @@ -7808,7 +7811,7 @@ dependencies = [ "sail-sql-analyzer", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", "xee-xpath", "xot", @@ -7816,7 +7819,7 @@ dependencies = [ [[package]] name = "sail-gold-test" -version = "0.6.4" +version = "0.7.0" dependencies = [ "clap", "lazy_static", @@ -7828,7 +7831,7 @@ dependencies = [ [[package]] name = "sail-iceberg" -version = "0.6.4" +version = "0.7.0" dependencies = [ "apache-avro", "arrow-schema", @@ -7856,6 +7859,8 @@ dependencies = [ "sail-common", "sail-common-datafusion", "sail-data-source", + "sail-logical-plan", + "sail-physical-plan", "sail-sql-analyzer", "serde", "serde_arrow", @@ -7866,7 +7871,7 @@ dependencies = [ [[package]] name = "sail-logical-optimizer" -version = "0.6.4" +version = "0.7.0" dependencies = [ "datafusion", "datafusion-common", @@ -7875,7 +7880,7 @@ dependencies = [ [[package]] name = "sail-logical-plan" -version = "0.6.4" +version = "0.7.0" dependencies = [ "comfy-table", "datafusion", @@ -7891,7 +7896,7 @@ dependencies = [ [[package]] name = "sail-object-store" -version = "0.6.4" +version = "0.7.0" dependencies = [ "async-stream", "async-trait", @@ -7913,10 +7918,9 @@ dependencies = [ "log", "object_store", "regex", - "reqwest", + "reqwest 0.12.28", "sail-common", - "sail-telemetry", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tonic", @@ -7925,7 +7929,7 @@ dependencies = [ [[package]] name = "sail-physical-optimizer" -version = "0.6.4" +version = "0.7.0" dependencies = [ "datafusion", "datafusion-physical-expr", @@ -7935,23 +7939,31 @@ dependencies = [ [[package]] name = "sail-physical-plan" -version = "0.6.4" +version = "0.7.0" dependencies = [ "datafusion", "datafusion-common", + "datafusion-datasource-parquet", "datafusion-expr", "futures", "lazy_static", + "log", + "object_store", + "parquet", + "sail-cache", "sail-catalog", "sail-common-datafusion", "sail-logical-plan", + "serde", + "serde_json", "tokio", "tokio-stream", + "uuid", ] [[package]] name = "sail-plan" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow", "arrow-schema", @@ -7974,9 +7986,11 @@ dependencies = [ "log", "object_store", "parquet-variant-compute", - "rand 0.10.1", + "rand 0.10.2", "regex", + "regex-syntax 0.8.11", "ryu", + "sail-cache", "sail-catalog", "sail-catalog-memory", "sail-common", @@ -7989,7 +8003,8 @@ dependencies = [ "sail-sql-analyzer", "serde", "serde_json", - "thiserror 2.0.18", + "serde_yaml", + "thiserror 2.0.19", "tokio", "tonic", "uuid", @@ -7997,7 +8012,7 @@ dependencies = [ [[package]] name = "sail-pyarrow" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow-array", "arrow-data", @@ -8007,7 +8022,7 @@ dependencies = [ [[package]] name = "sail-python" -version = "0.6.4" +version = "0.7.0" dependencies = [ "fastrace", "log", @@ -8022,7 +8037,7 @@ dependencies = [ [[package]] name = "sail-python-udf" -version = "0.6.4" +version = "0.7.0" dependencies = [ "datafusion", "datafusion-common", @@ -8035,41 +8050,27 @@ dependencies = [ "sail-common", "sail-common-datafusion", "sail-pyarrow", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", ] -[[package]] -name = "sail-server" -version = "0.6.4" -dependencies = [ - "axum", - "fastrace", - "log", - "sail-common", - "sail-telemetry", - "tokio", - "tonic", - "tonic-health", - "tonic-reflection", - "tower", -] - [[package]] name = "sail-session" -version = "0.6.4" +version = "0.7.0" dependencies = [ "async-trait", "chrono", "datafusion", "datafusion-common", + "datafusion-datasource", "datafusion-expr", "datafusion-physical-expr", "fastrace", "futures", "indexmap 2.14.0", "log", + "object_store", "readonly", "sail-cache", "sail-catalog", @@ -8092,18 +8093,18 @@ dependencies = [ "sail-physical-optimizer", "sail-physical-plan", "sail-plan", - "sail-server", "sail-sql-analyzer", - "sail-telemetry", "secrecy", - "thiserror 2.0.18", + "tempfile", + "thiserror 2.0.19", "tokio", "tonic", + "uuid", ] [[package]] name = "sail-spark-connect" -version = "0.6.4" +version = "0.7.0" dependencies = [ "arrow-schema", "async-stream", @@ -8132,13 +8133,12 @@ dependencies = [ "sail-execution", "sail-plan", "sail-python-udf", - "sail-server", "sail-session", "sail-sql-analyzer", "serde", "serde_json", - "syn 2.0.118", - "thiserror 2.0.18", + "syn 2.0.119", + "thiserror 2.0.19", "tokio", "tonic", "tonic-prost", @@ -8149,7 +8149,7 @@ dependencies = [ [[package]] name = "sail-sql-analyzer" -version = "0.6.4" +version = "0.7.0" dependencies = [ "chrono", "chumsky 0.12.0", @@ -8158,21 +8158,21 @@ dependencies = [ "regex", "sail-common", "sail-sql-parser", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] name = "sail-sql-macro" -version = "0.6.4" +version = "0.7.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "sail-sql-parser" -version = "0.6.4" +version = "0.7.0" dependencies = [ "chumsky 0.12.0", "either", @@ -8184,12 +8184,12 @@ dependencies = [ "sail-common", "sail-sql-macro", "serde", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "sail-telemetry" -version = "0.6.4" +version = "0.7.0" dependencies = [ "datafusion", "env_logger", @@ -8214,11 +8214,9 @@ dependencies = [ "sail-common-datafusion", "serde", "serde_yaml", - "syn 2.0.118", - "thiserror 2.0.18", + "syn 2.0.119", + "thiserror 2.0.19", "tokio", - "tonic", - "tower", ] [[package]] @@ -8260,7 +8258,7 @@ checksum = "d0778d6e209051bc4e75acfe83bcd7848601ec3dbe9c3dbb982829020e9128af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -8282,20 +8280,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "schemars" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" -dependencies = [ - "chrono", - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", - "uuid", -] - [[package]] name = "schemars" version = "0.9.0" @@ -8320,18 +8304,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "schemars_derive" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.118", -] - [[package]] name = "scoped-tls" version = "1.0.1" @@ -8366,7 +8338,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -8390,7 +8362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -8398,10 +8370,6 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "seq-macro" @@ -8411,9 +8379,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -8456,40 +8424,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "serde_derive_internals" -version = "0.29.1" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -8517,7 +8474,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8538,18 +8495,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_tokenstream" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c49585c52c01f13c5c2ebb333f14f6885d76daa768d8a037d28017ec538c69" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.118", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -8591,7 +8536,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8725,7 +8670,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8754,7 +8699,7 @@ dependencies = [ "simdutf8", "sonic-number", "sonic-simd", - "thiserror 2.0.18", + "thiserror 2.0.19", "zmij", ] @@ -8786,7 +8731,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8805,7 +8750,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8829,7 +8774,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8840,7 +8785,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8861,7 +8806,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8873,7 +8818,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8895,9 +8840,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -8932,7 +8888,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8941,7 +8897,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -8984,7 +8940,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -9016,7 +8972,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -9040,11 +8996,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.19", ] [[package]] @@ -9055,18 +9011,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -9165,9 +9121,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -9188,7 +9144,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9203,9 +9159,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -9369,7 +9325,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9407,7 +9363,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.118", + "syn 2.0.119", "tempfile", "tonic-build", ] @@ -9464,7 +9420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-util", "http 1.4.2", @@ -9510,7 +9466,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9543,7 +9499,7 @@ dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex-automata 0.4.14", + "regex-automata 0.4.16", "sharded-slab", "smallvec", "thread_local", @@ -9573,59 +9529,18 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "typify" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7144144e97e987c94758a3017c920a027feac0799df325d6df4fc8f08d02068e" -dependencies = [ - "typify-impl", - "typify-macro", -] - -[[package]] -name = "typify-impl" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062879d46aa4c9dfe0d33b035bbaf512da192131645d05deacb7033ec8581a09" -dependencies = [ - "heck", - "log", - "proc-macro2", - "quote", - "regress", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "syn 2.0.118", - "thiserror 2.0.18", - "unicode-ident", -] - -[[package]] -name = "typify-macro" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9708a3ceb6660ba3f8d2b8f0567e7d4b8b198e2b94d093b8a6077a751425de9e" -dependencies = [ - "proc-macro2", - "quote", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_tokenstream", - "syn 2.0.118", - "typify-impl", -] - [[package]] name = "ucd-trie" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "uncased" version = "0.9.10" @@ -9757,12 +9672,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" -[[package]] -name = "utf8-width" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" - [[package]] name = "utf8-zero" version = "0.8.1" @@ -9783,9 +9692,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -9837,7 +9746,7 @@ dependencies = [ "pin-project", "rand 0.9.4", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tower", @@ -9868,7 +9777,7 @@ dependencies = [ "quote", "serde", "serde_yaml", - "syn 2.0.118", + "syn 2.0.119", "tempfile", "url_path", "volo", @@ -9877,9 +9786,9 @@ dependencies = [ [[package]] name = "volo-thrift" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307c03439d0df02d6289dcddea8c1f4326cab0e6bd45b240da15b2afa711bafe" +checksum = "394e287a642d65e992d8115720036b51f08f59749c57b9889aa47fa5b5f7cc35" dependencies = [ "ahash 0.8.12", "anyhow", @@ -9899,7 +9808,7 @@ dependencies = [ "rustc-hash 2.1.2", "scopeguard", "sonic-rs", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "volo", @@ -9994,7 +9903,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -10094,7 +10003,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -10124,7 +10033,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10135,7 +10044,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10191,15 +10100,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" @@ -10521,7 +10421,7 @@ dependencies = [ "rust_decimal", "static_assertions", "strum", - "thiserror 2.0.18", + "thiserror 2.0.19", "v_jsonescape", "xee-name", "xee-schema-type", @@ -10578,7 +10478,7 @@ dependencies = [ "iri-string", "ordered-float 5.3.0", "rust_decimal", - "thiserror 2.0.18", + "thiserror 2.0.19", "xee-interpreter", "xee-ir", "xee-schema-type", @@ -10599,7 +10499,7 @@ dependencies = [ "ibig", "ordered-float 5.3.0", "rust_decimal", - "thiserror 2.0.18", + "thiserror 2.0.19", "xee-name", "xee-schema-type", "xee-xpath-lexer", @@ -10618,7 +10518,7 @@ dependencies = [ "iri-string", "ordered-float 5.3.0", "rust_decimal", - "thiserror 2.0.18", + "thiserror 2.0.19", "xee-interpreter", "xee-ir", "xee-schema-type", @@ -10646,7 +10546,7 @@ checksum = "30d233fc2ef8ab1a9f9bf7af9cc1c7f7e7d75eb9848f731a99d8f1d4c2259fe3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "xee-schema-type", "xee-xpath-ast", "xot", @@ -10726,7 +10626,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10738,7 +10638,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10759,7 +10659,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10779,7 +10679,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10841,7 +10741,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10852,7 +10752,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cb431daf6..17124768f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,14 +5,14 @@ members = [ resolver = "2" [workspace.package] -version = "0.6.4" +version = "0.7.0" authors = ["LakeSail "] -edition = "2021" +edition = "2024" homepage = "https://lakesail.com" license = "Apache-2.0" readme = "README.md" repository = "https://github.com/lakehq/sail" -rust-version = "1.95.0" +rust-version = "1.96.0" [workspace.lints.clippy] allow_attributes = "deny" @@ -46,6 +46,7 @@ serde_yaml = "0.9.34" serde_with = { version = "3.20.0", default-features = false, features = ["base64", "std", "macros"] } monostate = "1.0.2" regex = "1.12.3" +regex-syntax = "0.8.11" glob = "0.3.3" chrono = { version = "0.4.44", features = ["serde"] } chrono-tz = "0.10.4" @@ -79,13 +80,13 @@ num_enum = "0.7.6" num-traits = "0.2.19" log = "0.4.31" env_logger = "0.11.10" -fastrace = { version = "0.7.17", features = ["enable"] } -fastrace-opentelemetry = "0.16.0" -fastrace-futures = "0.7.17" -opentelemetry = "0.31.0" -opentelemetry_sdk = { version = "0.31.0", features = ["testing"] } -opentelemetry-otlp = { version = "0.31.0", features = ["tls", "tls-roots", "grpc-tonic"] } -opentelemetry-appender-log = "0.31.0" +fastrace = { version = "0.7.18", features = ["enable"] } +fastrace-opentelemetry = "0.18.1" +fastrace-futures = "0.7.18" +opentelemetry = "0.32.0" +opentelemetry_sdk = { version = "0.32.1", features = ["testing"] } +opentelemetry-otlp = { version = "0.32.0", features = ["tls", "tls-roots", "grpc-tonic", "http-json"] } +opentelemetry-appender-log = "0.32.0" figment = { version = "0.10.19", features = ["toml", "env"] } kube = "3.1.0" k8s-openapi = { version = "0.27.1", features = ["latest"] } @@ -118,8 +119,6 @@ apache-avro = { version = "0.21.0" } rust_decimal = "1.42.0" wiremock = "0.6.5" testcontainers = "0.26.2" -progenitor = "0.11.2" -progenitor-client = "0.11.2" secrecy = { version = "0.10.3", features = ["serde"] } tempfile = "3.27.0" readonly = "0.2.13" @@ -152,7 +151,7 @@ prost-types = "0.14" # The `axum` version must match the one used in `tonic` (replace `RELEASE` with the release we are using): # https://github.com/hyperium/tonic/blob/vRELEASE/tonic/Cargo.toml axum = "0.8.9" -datafusion = { version = "54.0.0", default-features = false, features = [ +datafusion = { version = "54.1.0", default-features = false, features = [ "nested_expressions", "crypto_expressions", "datetime_expressions", @@ -166,18 +165,19 @@ datafusion = { version = "54.0.0", default-features = false, features = [ "avro", "recursive_protection", ] } -datafusion-common = { version = "54.0.0", features = ["object_store"] } -datafusion-datasource = { version = "54.0.0" } -datafusion-datasource-avro = { version = "54.0.0" } -datafusion-datasource-json = { version = "54.0.0" } -datafusion-expr = { version = "54.0.0", default-features = false } -datafusion-expr-common = { version = "54.0.0" } -datafusion-proto = { version = "54.0.0" } -datafusion-functions = { version = "54.0.0" } -datafusion-functions-nested = { version = "54.0.0", default-features = false } -datafusion-physical-expr = { version = "54.0.0" } -datafusion-session = { version = "54.0.0" } -datafusion-spark = { version = "54.0.0", features = ["core"] } +datafusion-common = { version = "54.1.0", features = ["object_store"] } +datafusion-datasource = { version = "54.1.0" } +datafusion-datasource-avro = { version = "54.1.0" } +datafusion-datasource-json = { version = "54.1.0" } +datafusion-datasource-parquet = { version = "54.1.0" } +datafusion-expr = { version = "54.1.0", default-features = false } +datafusion-expr-common = { version = "54.1.0" } +datafusion-proto = { version = "54.1.0" } +datafusion-functions = { version = "54.1.0" } +datafusion-functions-nested = { version = "54.1.0", default-features = false } +datafusion-physical-expr = { version = "54.1.0" } +datafusion-session = { version = "54.1.0" } +datafusion-spark = { version = "54.1.0", features = ["core"] } pyo3 = { version = "0.29.0", features = ["serde"] } jiter = { version = "0.15.0", default-features = false } arrow = { version = "58.3.0", features = ["chrono-tz"] } diff --git a/crates/sail-build-scripts/Cargo.toml b/crates/sail-build-scripts/Cargo.toml index 88d468929..f2f20fdde 100644 --- a/crates/sail-build-scripts/Cargo.toml +++ b/crates/sail-build-scripts/Cargo.toml @@ -10,6 +10,7 @@ workspace = true lazy_static = { workspace = true } serde = { workspace = true } serde_yaml = { workspace = true } +thiserror = { workspace = true } quote = { workspace = true } proc-macro2 = { workspace = true } syn = { workspace = true } diff --git a/crates/sail-build-scripts/src/error.rs b/crates/sail-build-scripts/src/error.rs new file mode 100644 index 000000000..cb5236290 --- /dev/null +++ b/crates/sail-build-scripts/src/error.rs @@ -0,0 +1,15 @@ +#[derive(Debug, thiserror::Error)] +pub enum BuildError { + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("YAML error: {0}")] + Yaml(#[from] serde_yaml::Error), + #[error("environment variable error: {0}")] + EnvVar(#[from] std::env::VarError), + #[error("Rust syntax error: {0}")] + Syn(#[from] syn::Error), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +pub type BuildResult = Result; diff --git a/crates/sail-build-scripts/src/lib.rs b/crates/sail-build-scripts/src/lib.rs index 5b53f15a4..b51cf42cf 100644 --- a/crates/sail-build-scripts/src/lib.rs +++ b/crates/sail-build-scripts/src/lib.rs @@ -1 +1,3 @@ pub mod data_source; +pub mod error; +pub mod openapi; diff --git a/crates/sail-build-scripts/src/openapi/generator/config.rs b/crates/sail-build-scripts/src/openapi/generator/config.rs new file mode 100644 index 000000000..81229388a --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/generator/config.rs @@ -0,0 +1,22 @@ +use std::collections::BTreeSet; + +/// Configuration for OpenAPI code generation. +#[derive(Clone, Debug, Default)] +pub struct OpenApiConfig { + /// Schema names to omit from generated Rust type definitions. + /// + /// Use this when a schema needs a handwritten implementation in the crate that includes the + /// generated code. References to excluded schemas are still emitted by name. + pub excluded_schemas: BTreeSet, + /// Operation IDs to omit from generated `ApiClient` methods and operation error enums. + /// + /// This is useful for endpoints that are not needed by the target crate or that use currently + /// unsupported OpenAPI features. + pub excluded_operations: BTreeSet, +} + +impl OpenApiConfig { + pub fn new() -> Self { + Self::default() + } +} diff --git a/crates/sail-build-scripts/src/openapi/generator/core.rs b/crates/sail-build-scripts/src/openapi/generator/core.rs new file mode 100644 index 000000000..3163dded1 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/generator/core.rs @@ -0,0 +1,222 @@ +use std::path::Path; + +use proc_macro2::TokenStream; +use quote::quote; + +use super::config::OpenApiConfig; +use crate::error::{BuildError, BuildResult}; +use crate::openapi::generator::operation::OperationDefinition; +use crate::openapi::generator::schema::SchemaDefinition; +use crate::openapi::spec::{ + MaybeRef, OpenApi, Parameter, PathItem, RequestBody, Response, Schema, SchemaReference, + load_spec, +}; + +pub fn generate_openapi_client( + spec_path: impl AsRef, + output_path: impl AsRef, + config: OpenApiConfig, +) -> BuildResult<()> { + let spec_path = spec_path.as_ref(); + println!("cargo:rerun-if-changed={}", spec_path.display()); + let openapi = load_spec(spec_path)?; + let tokens = OpenApiGenerator::new(&openapi, config).generate_client()?; + std::fs::write(output_path, prettyplease::unparse(&syn::parse2(tokens)?))?; + Ok(()) +} + +/// OpenAPI code generator. +/// +/// The generator supports a deliberately small OpenAPI subset to cover services +/// used in the codebase. +pub(super) struct OpenApiGenerator<'a> { + pub(super) openapi: &'a OpenApi, + pub(super) config: OpenApiConfig, +} + +impl<'a> OpenApiGenerator<'a> { + pub(super) fn new(openapi: &'a OpenApi, config: OpenApiConfig) -> Self { + Self { openapi, config } + } + + pub(super) fn generate_client(&self) -> BuildResult { + let operations = self.operation_definitions()?; + let methods = operations + .iter() + .map(OperationDefinition::method_tokens) + .collect::, _>>()?; + let errors = operations + .iter() + .map(OperationDefinition::error_enum_tokens) + .collect::, _>>()?; + let schemas = self + .schema_definitions()? + .iter() + .map(SchemaDefinition::tokens) + .collect::>>()?; + + Ok(quote! { + #[derive(Debug)] + pub struct ApiClient { + pub base: String, + pub client: reqwest::Client, + pub headers: reqwest::header::HeaderMap, + } + + #[derive(Debug)] + pub struct Response { + pub inner: T, + pub status: reqwest::StatusCode, + pub headers: reqwest::header::HeaderMap, + } + + #[derive(Debug, thiserror::Error)] + pub enum ApiError { + #[error("request error: {0}")] + Request(#[from] reqwest::Error), + #[error("response error: status code {}", .0.status)] + Response(Response), + #[error("unknown error: status code {}", .0.status)] + Unknown(Response<()>), + } + + impl ApiError { + pub fn status(&self) -> Option { + match self { + Self::Request(error) => error.status(), + Self::Response(response) => Some(response.status), + Self::Unknown(response) => Some(response.status), + } + } + } + + fn eager_percent_encode(value: impl AsRef) -> String { + percent_encoding::utf8_percent_encode( + value.as_ref(), + percent_encoding::NON_ALPHANUMERIC, + ) + .to_string() + } + + impl ApiClient { + pub fn new( + base: impl Into, + client: reqwest::Client, + headers: reqwest::header::HeaderMap, + ) -> Self { + Self { + base: base.into(), + client, + headers, + } + } + + #(#methods)* + } + + #(#errors)* + #(#schemas)* + }) + } + + pub(super) fn resolve_path_item( + &self, + item: &'a MaybeRef, + ) -> BuildResult<&'a PathItem> { + match item { + MaybeRef::Value(value) => Ok(value), + MaybeRef::Ref(reference) => Err(BuildError::InvalidInput(format!( + "path item references are not supported: {}", + reference.reference + ))), + } + } + + pub(super) fn resolve_parameter( + &self, + parameter: &'a MaybeRef, + ) -> BuildResult<&'a Parameter> { + match parameter { + MaybeRef::Value(value) => Ok(value), + MaybeRef::Ref(reference) => { + let name = component_name(&reference.reference, "parameters")?; + self.openapi + .components + .parameters + .get(name) + .ok_or_else(|| { + BuildError::InvalidInput(format!("unknown parameter reference: {name}")) + }) + .and_then(|parameter| self.resolve_parameter(parameter)) + } + } + } + + pub(super) fn resolve_response( + &self, + response: &'a MaybeRef, + ) -> BuildResult<&'a Response> { + match response { + MaybeRef::Value(value) => Ok(value), + MaybeRef::Ref(reference) => { + let name = component_name(&reference.reference, "responses")?; + self.openapi + .components + .responses + .get(name) + .ok_or_else(|| { + BuildError::InvalidInput(format!("unknown response reference: {name}")) + }) + .and_then(|response| self.resolve_response(response)) + } + } + } + + pub(super) fn resolve_request_body( + &self, + body: &'a MaybeRef, + ) -> BuildResult<&'a RequestBody> { + match body { + MaybeRef::Value(value) => Ok(value), + MaybeRef::Ref(reference) => Err(BuildError::InvalidInput(format!( + "request body references are not supported: {}", + reference.reference + ))), + } + } + + pub(super) fn resolve_schema( + &self, + schema: &'a MaybeRef, + ) -> BuildResult<&'a Schema> { + match schema { + MaybeRef::Value(value) => Ok(value), + MaybeRef::Ref(reference) => { + let (_, schema) = self.resolve_schema_reference(&reference.reference)?; + Ok(schema) + } + } + } + + pub(super) fn resolve_schema_reference( + &self, + reference: &'a str, + ) -> BuildResult<(&'a str, &'a Schema)> { + let name = component_name(reference, "schemas")?; + let schema = + self.openapi.components.schemas.get(name).ok_or_else(|| { + BuildError::InvalidInput(format!("unknown schema reference: {name}")) + })?; + match schema { + MaybeRef::Value(value) => Ok((name, value)), + MaybeRef::Ref(reference) => self.resolve_schema_reference(&reference.reference), + } + } +} + +fn component_name<'a>(reference: &'a str, component: &str) -> BuildResult<&'a str> { + let prefix = format!("#/components/{component}/"); + reference + .strip_prefix(&prefix) + .ok_or_else(|| BuildError::InvalidInput(format!("unsupported reference: {reference}"))) +} diff --git a/crates/sail-build-scripts/src/openapi/generator/mod.rs b/crates/sail-build-scripts/src/openapi/generator/mod.rs new file mode 100644 index 000000000..0bddcc9a2 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/generator/mod.rs @@ -0,0 +1,7 @@ +mod config; +mod core; +mod operation; +mod schema; + +pub use self::config::OpenApiConfig; +pub use self::core::generate_openapi_client; diff --git a/crates/sail-build-scripts/src/openapi/generator/operation.rs b/crates/sail-build-scripts/src/openapi/generator/operation.rs new file mode 100644 index 000000000..2fa9b5bf3 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/generator/operation.rs @@ -0,0 +1,696 @@ +use proc_macro2::{Literal, TokenStream}; +use quote::quote; + +use super::core::OpenApiGenerator; +use super::schema::has_schema_type; +use crate::error::{BuildError, BuildResult}; +use crate::openapi::spec::{ + MaybeRef, MediaType, ObjectMap, Operation, ParameterLocation, PathItem, RequestBody, Response, + Schema, SchemaReference, SchemaType, +}; +use crate::openapi::utils::docs::doc_attrs; +use crate::openapi::utils::http::{HttpMethod, HttpStatus}; +use crate::openapi::utils::name::{RustName, type_name, value_name}; +use crate::openapi::utils::types::{RustType, TypePosition}; + +impl OpenApiGenerator<'_> { + pub(super) fn operation_definitions(&self) -> BuildResult> { + let mut output = Vec::new(); + for (path, item) in &self.openapi.paths { + let item = self.resolve_path_item(item)?; + for (method, operation) in Self::collect_operations(item) { + let operation_id = operation.operation_id.as_deref().ok_or_else(|| { + BuildError::InvalidInput(format!( + "operation {} {path} is missing operationId", + method.name() + )) + })?; + if self.config.excluded_operations.contains(operation_id) { + continue; + } + output.push(self.operation_definition( + method, + path, + operation_id, + item, + operation, + )?); + } + } + Ok(output) + } + + fn operation_definition( + &self, + method: HttpMethod, + path: &str, + operation_id: &str, + path_item: &PathItem, + operation: &Operation, + ) -> BuildResult { + let parameters = self.collect_parameters(operation_id, path_item, operation)?; + let body = operation + .request_body + .as_ref() + .map(|body| self.request_body_type(operation_id, body)) + .transpose()?; + Ok(OperationDefinition { + method, + path: path.to_owned(), + operation_id: operation_id.to_owned(), + summary: operation.summary.clone(), + description: operation.description.clone(), + success_response: self.success_response(operation_id, operation)?, + error_responses: self.error_responses(operation_id, operation)?, + parameters, + body, + }) + } + + fn collect_operations(path_item: &PathItem) -> Vec<(HttpMethod, &Operation)> { + [ + (HttpMethod::Get, path_item.get.as_ref()), + (HttpMethod::Put, path_item.put.as_ref()), + (HttpMethod::Post, path_item.post.as_ref()), + (HttpMethod::Delete, path_item.delete.as_ref()), + (HttpMethod::Options, path_item.options.as_ref()), + (HttpMethod::Head, path_item.head.as_ref()), + (HttpMethod::Patch, path_item.patch.as_ref()), + (HttpMethod::Trace, path_item.trace.as_ref()), + ] + .into_iter() + .filter_map(|(method, operation)| operation.map(|operation| (method, operation))) + .collect() + } + + fn collect_parameters( + &self, + operation_id: &str, + path_item: &PathItem, + operation: &Operation, + ) -> BuildResult> { + let mut output = Vec::new(); + for parameter in path_item + .parameters + .iter() + .chain(operation.parameters.iter()) + { + let parameter = self.resolve_parameter(parameter)?; + let location = parameter.location.clone(); + let schema = parameter.schema.as_ref().ok_or_else(|| { + BuildError::InvalidInput(format!( + "parameter {} in operation {} is missing a schema", + parameter.name, operation_id + )) + })?; + let required = if matches!(location, ParameterLocation::Path) { + // The OpenAPI standard requires path parameters to have `required: true`. + // The Iceberg REST catalog specification intentionally marks the `{prefix}` path + // parameter as `required: false`, so honor an explicit `false` here. + parameter.required.unwrap_or(true) + } else { + parameter.required == Some(true) + }; + let schema_type = self.parameter_type(schema)?; + let is_array = schema_type.is_vec(); + let rust_type = if required { + schema_type.clone() + } else { + RustType::Option(Box::new(schema_type)) + }; + let kind = match location { + ParameterLocation::Path => ParameterKind::Path { required }, + ParameterLocation::Query => ParameterKind::Query { + required, + array: is_array, + }, + ParameterLocation::Header => ParameterKind::Header { + required, + array: is_array, + }, + ParameterLocation::Cookie => { + return Err(BuildError::InvalidInput(format!( + "operation {} contains unsupported {:?} parameter {}", + operation_id, parameter.location, parameter.name + ))); + } + }; + let identifier = if matches!(kind, ParameterKind::Header { .. }) { + parameter + .name + .strip_prefix("X-") + .or_else(|| parameter.name.strip_prefix("x-")) + .unwrap_or(¶meter.name) + } else { + ¶meter.name + }; + output.push(OperationParameter { + name: parameter.name.clone(), + identifier: value_name(identifier), + rust_type, + kind, + }); + } + Ok(output) + } + + fn request_body_type( + &self, + operation_id: &str, + body: &MaybeRef, + ) -> BuildResult { + let body = self.resolve_request_body(body)?; + let schema = json_content_schema(&body.content)?.ok_or_else(|| { + BuildError::InvalidInput(format!( + "operation {operation_id} has request body without JSON schema" + )) + })?; + Ok(OperationBody { + rust_type: self.schema_type(schema, TypePosition::Normal)?, + }) + } + + fn success_response( + &self, + operation_id: &str, + operation: &Operation, + ) -> BuildResult { + let mut response = None; + for (status, item) in &operation.responses { + let status = HttpStatus::parse(status)?; + if matches!(status, HttpStatus::Success(_)) { + if response.is_some() { + return Err(BuildError::InvalidInput(format!( + "operation {operation_id} defines multiple success responses" + ))); + } + let item = self.resolve_response(item)?; + response = Some(OperationResponse { + status, + rust_type: self.response_type(item)?, + }); + } + } + response.ok_or_else(|| { + BuildError::InvalidInput(format!( + "operation {operation_id} does not define a success response" + )) + }) + } + + fn error_responses( + &self, + _operation_id: &str, + operation: &Operation, + ) -> BuildResult> { + let mut output = Vec::new(); + for (status, response) in &operation.responses { + let status = HttpStatus::parse(status)?; + if matches!(status, HttpStatus::Success(_)) { + continue; + } + let response = self.resolve_response(response)?; + output.push(OperationResponse { + status, + rust_type: self.response_type(response)?, + }); + } + Ok(output) + } + + fn response_type(&self, response: &Response) -> BuildResult { + json_content_schema(&response.content)? + .map(|schema| self.schema_type(schema, TypePosition::Normal)) + .transpose() + .map(|value| value.unwrap_or(RustType::Unit)) + } + + fn parameter_type(&self, schema: &MaybeRef) -> BuildResult { + let schema = match schema { + MaybeRef::Value(schema) => schema, + MaybeRef::Ref(reference) => { + let (_, schema) = self.resolve_schema_reference(&reference.reference)?; + if self.is_named_parameter_schema(schema)? { + return self + .schema_type(&MaybeRef::Ref(reference.clone()), TypePosition::Normal); + } + schema + } + }; + + if has_schema_type(schema, SchemaType::Array) { + let items = schema.items.as_deref().ok_or_else(|| { + BuildError::InvalidInput("array parameter schema is missing items".to_owned()) + })?; + let item = self.parameter_type(items)?; + Ok(RustType::Vec(Box::new(item))) + } else if has_schema_type(schema, SchemaType::Boolean) { + Ok(RustType::Bool) + } else if has_schema_type(schema, SchemaType::Integer) { + Ok(match schema.format.as_deref() { + Some("int64") => RustType::I64, + _ => RustType::I32, + }) + } else if has_schema_type(schema, SchemaType::Number) { + Ok(match schema.format.as_deref() { + Some("float") => RustType::F32, + Some("double") | None => RustType::F64, + _ => RustType::F64, + }) + } else if has_schema_type(schema, SchemaType::String) { + Ok(RustType::String) + } else { + Err(BuildError::InvalidInput( + "path, query, and header parameters must be primitive JSON values".to_owned(), + )) + } + } +} + +pub(super) struct OperationDefinition { + method: HttpMethod, + path: String, + operation_id: String, + summary: Option, + description: Option, + parameters: Vec, + body: Option, + success_response: OperationResponse, + error_responses: Vec, +} + +impl OperationDefinition { + pub(super) fn method_tokens(&self) -> BuildResult { + let method_name = value_name(&self.operation_id); + let error_type = operation_error_type_name(&self.operation_id); + let path = generate_path_expression(&self.path, &self.parameters)?; + let method = RustName::new(self.method.name()); + let docs = doc_attrs(self.summary.as_deref(), self.description.as_deref()); + + let arguments = self.parameters.iter().map(|parameter| { + let ident = ¶meter.identifier; + let rust_type = ¶meter.rust_type; + quote! { #ident: #rust_type } + }); + let body_argument = self.body.as_ref().map(|body| { + let rust_type = &body.rust_type; + quote! { body: #rust_type } + }); + let query = generate_query(&self.parameters); + let headers = generate_headers(&self.parameters); + let has_query = !query.is_empty(); + let has_headers = !headers.is_empty(); + let mutability = if has_query || has_headers || self.body.is_some() { + quote! { mut } + } else { + quote! {} + }; + let request = quote! { + let #mutability request = self + .client + .request(reqwest::Method::#method, url) + .headers(self.headers.clone()); + }; + let query_request = has_query.then(|| { + quote! { + if !query.is_empty() { + request = request.query(&query); + } + } + }); + let body_request = self.body.is_some().then(|| { + quote! { + request = request.json(&body); + } + }); + let success_type = &self.success_response.rust_type; + let success_status = match self.success_response.status { + HttpStatus::Success(status) => status, + HttpStatus::ExactError(status) => { + return Err(BuildError::InvalidInput(format!( + "success response must use a success HTTP status, got {status}" + ))); + } + HttpStatus::ClientError | HttpStatus::ServerError => { + return Err(BuildError::InvalidInput( + "success response must use an exact HTTP status, got range".to_owned(), + )); + } + }; + let success_body = generate_response_body_expression(success_type); + let success_status_check = if self.success_response.rust_type.is_unit() { + quote! { status.is_success() } + } else { + quote! { status.as_u16() == #success_status } + }; + let error_body = if self.error_responses.is_empty() { + quote! { + response.error_for_status_ref()?; + Err(ApiError::Unknown(Response { + inner: (), + status, + headers, + })) + } + } else { + let arms = self + .error_responses + .iter() + .map(|response| generate_error_match_arm(&error_type, response)) + .collect::>>()?; + quote! { + let inner = match status.as_u16() { + #(#arms)* + _ => { + response.error_for_status_ref()?; + return Err(ApiError::Unknown(Response { + inner: (), + status, + headers, + })); + } + }; + Err(ApiError::Response(Response { + inner, + status, + headers, + })) + } + }; + + Ok(quote! { + #(#docs)* + pub async fn #method_name( + &self, + #(#arguments,)* + #body_argument + ) -> Result, ApiError<#error_type>> { + let path = #path; + let url = format!("{}{}", self.base.trim_end_matches('/'), path); + #(#query)* + #request + #(#headers)* + #query_request + #body_request + let response = request.send().await?; + let status = response.status(); + let headers = response.headers().clone(); + if #success_status_check { + let inner = #success_body; + return Ok(Response { + inner, + status, + headers, + }); + } + #error_body + } + }) + } + + pub(super) fn error_enum_tokens(&self) -> BuildResult { + let name = operation_error_type_name(&self.operation_id); + let variants = self + .error_responses + .iter() + .map(|response| { + let variant = type_name(response.status.variant()?); + if response.rust_type.is_unit() { + Ok(quote! { #variant, }) + } else { + let rust_type = &response.rust_type; + Ok(quote! { #variant(#rust_type), }) + } + }) + .collect::>>()?; + Ok(quote! { + #[derive(Debug)] + pub enum #name { + #(#variants)* + } + }) + } +} + +fn is_json_media_type(media_type: &str) -> bool { + media_type == "application/json" || media_type.ends_with("+json") +} + +fn json_content_schema( + content: &ObjectMap, +) -> BuildResult>> { + if content.is_empty() { + return Ok(None); + } + let mut schema = None; + for (media_type, value) in content { + if !is_json_media_type(media_type) { + return Err(BuildError::InvalidInput(format!( + "unsupported media type: {media_type}" + ))); + } + schema = value.schema.as_ref(); + } + Ok(schema) +} + +fn generate_path_expression( + path: &str, + parameters: &[OperationParameter], +) -> BuildResult { + let mut expression = String::new(); + let mut rest = path; + let mut arguments = Vec::new(); + while let Some(start) = rest.find('{') { + let (before, after_start) = rest.split_at(start); + expression.push_str(before); + let Some(end) = after_start.find('}') else { + return Err(BuildError::InvalidInput(format!( + "path contains unclosed parameter: {path}" + ))); + }; + let name = &after_start[1..end]; + let parameter = parameters + .iter() + .find(|parameter| { + matches!(parameter.kind, ParameterKind::Path { .. }) && parameter.name == name + }) + .ok_or_else(|| { + BuildError::InvalidInput(format!( + "path parameter {name} in {path} has no parameter definition" + )) + })?; + let ident = ¶meter.identifier; + let required = parameter.kind.required(); + let optional_segment = + !required && expression.ends_with('/') && after_start[end + 1..].starts_with('/'); + if optional_segment { + expression.pop(); + } + expression.push_str("{}"); + let argument = if !required { + let format = if optional_segment { "/{}" } else { "{}" }; + let value = if parameter.rust_type.is_string() { + quote! { value.as_str() } + } else { + quote! { value.to_string() } + }; + quote! { + #ident + .as_ref() + .map(|value| format!(#format, eager_percent_encode(#value))) + .unwrap_or_default() + } + } else { + let value = if parameter.rust_type.is_string() { + quote! { #ident.as_str() } + } else { + quote! { #ident.to_string() } + }; + quote! { eager_percent_encode(#value) } + }; + arguments.push(argument); + rest = &after_start[end + 1..]; + } + expression.push_str(rest); + if arguments.is_empty() { + Ok(quote! { #expression }) + } else { + Ok(quote! { format!(#expression, #(#arguments),*) }) + } +} + +fn generate_query(parameters: &[OperationParameter]) -> Vec { + let mut output = Vec::new(); + if parameters + .iter() + .any(|parameter| matches!(parameter.kind, ParameterKind::Query { .. })) + { + output.push(quote! { let mut query = Vec::<(String, String)>::new(); }); + } + for parameter in parameters { + let ParameterKind::Query { required, array } = parameter.kind else { + continue; + }; + let name = ¶meter.name; + let identifier = ¶meter.identifier; + output.push(match (required, array) { + (true, false) => { + quote! { query.push((#name.to_owned(), #identifier.to_string())); } + } + (false, false) => { + quote! { + if let Some(value) = #identifier.as_ref() { + query.push((#name.to_owned(), value.to_string())); + } + } + } + (true, true) => { + quote! { + for value in &#identifier { + query.push((#name.to_owned(), value.to_string())); + } + } + } + (false, true) => { + quote! { + if let Some(values) = #identifier.as_ref() { + for value in values { + query.push((#name.to_owned(), value.to_string())); + } + } + } + } + }); + } + output +} + +fn generate_headers(parameters: &[OperationParameter]) -> Vec { + let mut output = Vec::new(); + for parameter in parameters { + let ParameterKind::Header { required, array } = parameter.kind else { + continue; + }; + let name = ¶meter.name; + let identifier = ¶meter.identifier; + output.push(match (required, array) { + (true, false) => { + quote! { request = request.header(#name, #identifier.to_string()); } + } + (false, false) => { + quote! { + if let Some(value) = #identifier.as_ref() { + request = request.header(#name, value.to_string()); + } + } + } + (true, true) => { + quote! { + let value = #identifier + .iter() + .map(std::string::ToString::to_string) + .collect::>() + .join(","); + request = request.header(#name, value); + } + } + (false, true) => { + quote! { + if let Some(values) = #identifier.as_ref() { + let value = values + .iter() + .map(std::string::ToString::to_string) + .collect::>() + .join(","); + request = request.header(#name, value); + } + } + } + }); + } + output +} + +fn generate_response_body_expression(rust_type: &RustType) -> TokenStream { + if rust_type.is_unit() { + quote! { () } + } else { + quote! { + match response.json::<#rust_type>().await { + Ok(value) => value, + Err(_) => { + return Err(ApiError::Unknown(Response { + inner: (), + status, + headers, + })); + } + } + } + } +} + +fn generate_error_match_arm( + error_type: &RustName, + response: &OperationResponse, +) -> BuildResult { + let variant = type_name(response.status.variant()?); + let value = if response.rust_type.is_unit() { + quote! { #error_type::#variant } + } else { + let body = generate_response_body_expression(&response.rust_type); + quote! { #error_type::#variant(#body) } + }; + if let Some((start, end)) = response.status.range() { + let start = Literal::u16_unsuffixed(start); + let end = Literal::u16_unsuffixed(end); + Ok(quote! { #start..=#end => #value, }) + } else if let HttpStatus::ExactError(status) = response.status { + let status = Literal::u16_unsuffixed(status); + Ok(quote! { #status => #value, }) + } else { + Err(BuildError::InvalidInput(format!( + "error response must use an exact HTTP status or range, got {:?}", + response.status + ))) + } +} + +fn operation_error_type_name(operation_id: &str) -> RustName { + RustName::new(format!("{}Error", type_name(operation_id))) +} + +#[derive(Clone)] +struct OperationParameter { + name: String, + identifier: RustName, + rust_type: RustType, + kind: ParameterKind, +} + +#[derive(Clone, Copy)] +enum ParameterKind { + Path { required: bool }, + Query { required: bool, array: bool }, + Header { required: bool, array: bool }, +} + +impl ParameterKind { + fn required(&self) -> bool { + match self { + Self::Path { required } + | Self::Query { required, .. } + | Self::Header { required, .. } => *required, + } + } +} + +struct OperationBody { + rust_type: RustType, +} + +struct OperationResponse { + status: HttpStatus, + rust_type: RustType, +} diff --git a/crates/sail-build-scripts/src/openapi/generator/schema.rs b/crates/sail-build-scripts/src/openapi/generator/schema.rs new file mode 100644 index 000000000..021404e19 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/generator/schema.rs @@ -0,0 +1,961 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use proc_macro2::TokenStream; +use quote::quote; + +use super::core::OpenApiGenerator; +use crate::error::{BuildError, BuildResult}; +use crate::openapi::spec::{AdditionalProperties, MaybeRef, Schema, SchemaReference, SchemaType}; +use crate::openapi::utils::docs::doc_attrs; +use crate::openapi::utils::name::{RustName, type_name, value_name}; +use crate::openapi::utils::types::{RustType, TypePosition}; + +impl<'a> OpenApiGenerator<'a> { + pub(super) fn schema_definitions(&self) -> BuildResult> { + self.openapi + .components + .schemas + .iter() + .filter(|(name, _)| !self.config.excluded_schemas.contains(name.as_str())) + .map(|(name, schema)| self.schema_definition(name, schema)) + .collect() + } + + fn schema_definition( + &self, + name: &str, + schema: &'a MaybeRef, + ) -> BuildResult { + let mut inline = InlineSchemas::new(name); + let schema = self.resolve_schema(schema)?; + let schema = self.schema_definition_inner(type_name(name), schema, &mut inline, false)?; + Ok(SchemaDefinition { + module_name: inline.module_name(), + inline_definitions: inline.definitions, + ..schema + }) + } + + fn schema_definition_inner( + &self, + type_name: RustName, + schema: &'a Schema, + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult { + if let Some(variants) = string_enum_variants(schema)? { + return Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Enum { + tag: None, + untagged: false, + variants, + }, + )); + } + + if let Some(tag) = discriminator_tag(schema) { + let variants = self.discriminator_variants(schema, inline, in_module)?; + return Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Enum { + tag: Some(tag.to_owned()), + untagged: false, + variants, + }, + )); + } + + if !schema.one_of.is_empty() || !schema.any_of.is_empty() { + if !schema.properties.is_empty() { + return Err(BuildError::InvalidInput(format!( + "anyOf or oneOf schemas with sibling properties are unsupported: {type_name}" + ))); + } + let schemas = schema + .one_of + .iter() + .chain(schema.any_of.iter()) + .collect::>(); + if let Some((tag, variants)) = + self.inferred_tagged_enum_variants(&schemas, inline, in_module)? + { + return Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Enum { + tag: Some(tag), + untagged: false, + variants, + }, + )); + } + // OpenAPI `anyOf` can legitimately match more than one variant, so a discriminator is + // not always enough to make it equivalent to a Rust enum. Keep this fallback untagged + // unless the variants can be proven to inherit a common discriminator. + let variants = schemas + .iter() + .map(|schema| self.schema_enum_variant(schema, None)) + .collect::, _>>()?; + return Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Enum { + tag: None, + untagged: true, + variants, + }, + )); + } + + let fields = self.collect_object_fields(schema, inline, in_module)?; + if !fields.is_empty() { + return Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Struct { fields }, + )); + } + + if let Some(map_type) = self.map_type_with_inline(schema, inline, in_module, "Value")? { + return Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Transparent { + rust_type: map_type, + }, + )); + } + + let rust_type = self.schema_type_inner_with_inline( + schema, + TypePosition::Nested, + inline, + in_module, + "Item", + )?; + Ok(SchemaDefinition::new( + type_name, + schema, + SchemaKind::Transparent { rust_type }, + )) + } + + pub(super) fn schema_type( + &self, + schema: &'a MaybeRef, + position: TypePosition, + ) -> BuildResult { + let schema = match schema { + MaybeRef::Value(schema) => schema, + MaybeRef::Ref(reference) => { + return self.schema_reference_type(&reference.reference, position); + } + }; + let mut rust_type = self.schema_type_inner(schema, position)?; + if is_nullable(schema) { + rust_type = RustType::Option(Box::new(rust_type)); + } + Ok(rust_type) + } + + fn schema_type_inner( + &self, + schema: &'a Schema, + position: TypePosition, + ) -> BuildResult { + if let Some(item) = transparent_all_of(schema) { + return self.schema_type(item, position); + } + if has_schema_type(schema, SchemaType::Array) { + let items = schema.items.as_deref().ok_or_else(|| { + BuildError::InvalidInput("array schema is missing items".to_owned()) + })?; + let item = self.schema_type(items, TypePosition::Normal)?; + return Ok(RustType::Vec(Box::new(item))); + } + if let Some(map_type) = self.map_type(schema)? { + return Ok(map_type); + } + if let Some(rust_type) = primitive_schema_type(schema) { + return Ok(rust_type); + } + if has_schema_type(schema, SchemaType::Object) { + return Ok(RustType::JsonValue); + } + Ok(RustType::Unit) + } + + fn schema_type_with_inline( + &self, + schema: &'a MaybeRef, + position: TypePosition, + inline: &mut InlineSchemas, + in_module: bool, + suggested_name: &str, + ) -> BuildResult { + let schema = match schema { + MaybeRef::Value(schema) => schema, + MaybeRef::Ref(reference) => { + return self.schema_reference_type(&reference.reference, position); + } + }; + let mut rust_type = self.schema_type_inner_with_inline( + schema, + position, + inline, + in_module, + suggested_name, + )?; + if is_nullable(schema) { + rust_type = RustType::Option(Box::new(rust_type)); + } + Ok(rust_type) + } + + fn schema_type_inner_with_inline( + &self, + schema: &'a Schema, + position: TypePosition, + inline: &mut InlineSchemas, + in_module: bool, + suggested_name: &str, + ) -> BuildResult { + if let Some(item) = transparent_all_of(schema) { + return self.schema_type_with_inline(item, position, inline, in_module, suggested_name); + } + if has_schema_type(schema, SchemaType::Array) { + let items = schema.items.as_deref().ok_or_else(|| { + BuildError::InvalidInput("array schema is missing items".to_owned()) + })?; + let item_name = if suggested_name == "Item" { + "Item".to_owned() + } else { + format!("{suggested_name}Item") + }; + let item = self.schema_type_with_inline( + items, + TypePosition::Normal, + inline, + in_module, + &item_name, + )?; + return Ok(RustType::Vec(Box::new(item))); + } + if should_define_inline_schemas(schema) { + return self.define_inline_schemas(schema, inline, suggested_name, in_module); + } + if let Some(map_type) = + self.map_type_with_inline(schema, inline, in_module, suggested_name)? + { + return Ok(map_type); + } + if let Some(rust_type) = primitive_schema_type(schema) { + return Ok(rust_type); + } + if has_schema_type(schema, SchemaType::Object) { + return Ok(RustType::JsonValue); + } + Ok(RustType::Unit) + } + + fn schema_reference_type( + &self, + reference: &str, + position: TypePosition, + ) -> BuildResult { + let (name, _) = self.resolve_schema_reference(reference)?; + let rust_type = RustType::Named { + qualifier: Vec::new(), + name: type_name(name), + }; + Ok(match position { + TypePosition::Normal => rust_type, + TypePosition::Nested => RustType::Box(Box::new(rust_type)), + }) + } + + fn define_inline_schemas( + &self, + schema: &'a Schema, + inline: &mut InlineSchemas, + suggested_name: &str, + in_module: bool, + ) -> BuildResult { + let type_name = type_name(suggested_name); + let definition = self.schema_definition_inner(type_name.clone(), schema, inline, true)?; + inline.definitions.push(definition); + if in_module { + Ok(RustType::Named { + qualifier: Vec::new(), + name: type_name, + }) + } else { + Ok(RustType::Named { + qualifier: vec![inline.module_name.clone()], + name: type_name, + }) + } + } + + fn map_type(&self, schema: &'a Schema) -> BuildResult> { + let Some(additional_properties) = &schema.additional_properties else { + return Ok(None); + }; + let value_type = match additional_properties { + AdditionalProperties::Bool(true) => RustType::JsonValue, + AdditionalProperties::Bool(false) => return Ok(None), + AdditionalProperties::Schema(schema) => { + self.schema_type(schema, TypePosition::Normal)? + } + }; + Ok(Some(RustType::Map(Box::new(value_type)))) + } + + fn map_type_with_inline( + &self, + schema: &'a Schema, + inline: &mut InlineSchemas, + in_module: bool, + suggested_name: &str, + ) -> BuildResult> { + let Some(additional_properties) = &schema.additional_properties else { + return Ok(None); + }; + let value_type = match additional_properties { + AdditionalProperties::Bool(true) => RustType::JsonValue, + AdditionalProperties::Bool(false) => return Ok(None), + AdditionalProperties::Schema(schema) => self.schema_type_with_inline( + schema, + TypePosition::Normal, + inline, + in_module, + &format!("{suggested_name}Value"), + )?, + }; + Ok(Some(RustType::Map(Box::new(value_type)))) + } + + fn collect_object_fields( + &self, + schema: &'a Schema, + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult> { + let mut properties = BTreeMap::new(); + let mut required = BTreeSet::new(); + self.collect_object_fields_inner(schema, &mut properties, &mut required)?; + let mut output = Vec::new(); + for (name, schema) in properties { + let is_required = required.contains(name.as_str()); + let identifier = value_name(&name); + let rename = + (identifier.to_string().trim_start_matches("r#") != name).then(|| name.clone()); + let rust_type = self.schema_type_with_inline( + schema, + TypePosition::Nested, + inline, + in_module, + &type_name(&name).to_string(), + )?; + let rust_type = if is_required || rust_type.is_option() { + rust_type + } else { + RustType::Option(Box::new(rust_type)) + }; + output.push(SchemaField { + name, + identifier, + rename, + is_required, + rust_type, + }); + } + Ok(output) + } + + fn collect_object_fields_inner( + &self, + schema: &'a Schema, + properties: &mut BTreeMap>, + required: &mut BTreeSet, + ) -> BuildResult<()> { + for item in &schema.all_of { + let schema = self.resolve_schema(item)?; + self.collect_object_fields_inner(schema, properties, required)?; + } + for value in &schema.required { + required.insert(value.clone()); + } + for (name, schema) in &schema.properties { + properties.insert(name.clone(), schema); + } + Ok(()) + } + + fn schema_enum_variant( + &self, + schema: &'a MaybeRef, + rename: Option, + ) -> BuildResult { + let MaybeRef::Ref(reference) = schema else { + return Err(BuildError::InvalidInput( + "enum variant with inline schema is unsupported".to_string(), + )); + }; + let (name, _) = self.resolve_schema_reference(&reference.reference)?; + let variant = type_name(name); + let rust_type = self.schema_type(schema, TypePosition::Nested)?; + Ok(EnumVariant { + name: variant, + rename, + kind: EnumVariantKind::Tuple { rust_type }, + }) + } + + fn discriminator_variants( + &self, + schema: &'a Schema, + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult> { + let discriminator = schema.discriminator.as_ref().ok_or_else(|| { + BuildError::InvalidInput("schema is missing discriminator".to_owned()) + })?; + if !schema.any_of.is_empty() { + return Err(BuildError::InvalidInput( + "discriminator anyOf variants are unsupported".to_owned(), + )); + } + let variants = if !discriminator.mapping.is_empty() { + self.discriminator_variants_from_mapping(discriminator, None, inline, in_module)? + } else if !schema.one_of.is_empty() { + self.discriminator_variants_from_schemas( + discriminator, + &schema.one_of, + inline, + in_module, + )? + } else { + Vec::new() + }; + Ok(variants) + } + + fn discriminator_variants_from_mapping( + &self, + discriminator: &crate::openapi::spec::Discriminator, + references: Option<&BTreeSet<&str>>, + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult> { + discriminator + .mapping + .iter() + .filter(|(_, reference)| { + references.is_none_or(|references| references.contains(reference.as_str())) + }) + .map(|(value, reference)| { + self.discriminator_variant( + value.clone(), + reference, + &discriminator.property_name, + inline, + in_module, + ) + }) + .collect() + } + + fn discriminator_variants_from_schemas( + &self, + discriminator: &crate::openapi::spec::Discriminator, + schemas: &'a [MaybeRef], + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult> { + schemas + .iter() + .map(|schema| { + let MaybeRef::Ref(reference) = schema else { + return Err(BuildError::InvalidInput( + "discriminator variants must be schema references".to_owned(), + )); + }; + let reference = &reference.reference; + let (name, _) = self.resolve_schema_reference(reference)?; + self.discriminator_variant( + name.to_owned(), + reference, + &discriminator.property_name, + inline, + in_module, + ) + }) + .collect() + } + + fn discriminator_variant( + &self, + value: String, + reference: &str, + tag: &str, + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult { + let (_, schema) = self.resolve_schema_reference(reference)?; + let fields = self + .collect_object_fields(schema, inline, in_module)? + .into_iter() + .filter(|field| field.name != tag) + .collect::>(); + let kind = if fields.is_empty() { + EnumVariantKind::Unit + } else { + EnumVariantKind::Struct { fields } + }; + Ok(EnumVariant { + name: type_name(&value), + rename: Some(value), + kind, + }) + } + + fn inferred_tagged_enum_variants( + &self, + schemas: &[&'a MaybeRef], + inline: &mut InlineSchemas, + in_module: bool, + ) -> BuildResult)>> { + if schemas.is_empty() { + return Ok(None); + } + + let mut inherited_discriminator = None; + let mut references = BTreeSet::new(); + for schema in schemas { + let MaybeRef::Ref(reference) = *schema else { + return Ok(None); + }; + let (_, schema) = self.resolve_schema_reference(&reference.reference)?; + let Some((base_reference, discriminator)) = self.inherited_discriminator(schema)? + else { + return Ok(None); + }; + match inherited_discriminator { + None => inherited_discriminator = Some((base_reference, discriminator)), + Some((existing, _)) if existing == base_reference => {} + Some(_) => return Ok(None), + } + references.insert(reference.reference.as_str()); + } + + let Some((_, discriminator)) = inherited_discriminator else { + return Ok(None); + }; + if discriminator.mapping.is_empty() { + return Ok(None); + } + let tag = discriminator.property_name.clone(); + let variants = self.discriminator_variants_from_mapping( + discriminator, + Some(&references), + inline, + in_module, + )?; + + Ok(Some((tag, variants))) + } + + fn inherited_discriminator( + &self, + schema: &'a Schema, + ) -> BuildResult> { + let mut output = None; + for item in &schema.all_of { + let MaybeRef::Ref(reference) = item else { + continue; + }; + let (_, schema) = self.resolve_schema_reference(&reference.reference)?; + if discriminator_tag(schema).is_some() { + if output.is_some() { + return Ok(None); + } + output = Some((reference.reference.as_str(), schema.discriminator.as_ref())); + } + } + Ok(output.and_then(|(reference, discriminator)| { + discriminator.map(|discriminator| (reference, discriminator)) + })) + } + + pub(super) fn is_named_parameter_schema(&self, schema: &Schema) -> BuildResult { + if string_enum_variants(schema)?.is_some() || primitive_schema_type(schema).is_some() { + return Ok(true); + } + if let Some(schema) = transparent_all_of(schema) { + return self + .resolve_schema(schema) + .and_then(|schema| self.is_named_parameter_schema(schema)); + } + Ok(false) + } +} + +pub(super) fn has_schema_type(schema: &Schema, schema_type: SchemaType) -> bool { + schema + .r#type + .as_ref() + .is_some_and(|types| types.contains(&schema_type)) +} + +fn primitive_schema_type(schema: &Schema) -> Option { + if has_schema_type(schema, SchemaType::Boolean) { + return Some(RustType::Bool); + } + if has_schema_type(schema, SchemaType::Integer) { + return Some(match schema.format.as_deref() { + Some("int64") => RustType::I64, + _ => RustType::I32, + }); + } + if has_schema_type(schema, SchemaType::Number) { + return Some(match schema.format.as_deref() { + Some("float") => RustType::F32, + Some("double") | None => RustType::F64, + _ => RustType::F64, + }); + } + if has_schema_type(schema, SchemaType::String) { + return Some(RustType::String); + } + None +} + +fn is_nullable(schema: &Schema) -> bool { + schema.nullable == Some(true) || has_schema_type(schema, SchemaType::Null) +} + +fn string_enum_variants(schema: &Schema) -> BuildResult>> { + if schema.enum_values.is_empty() || !has_schema_type(schema, SchemaType::String) { + return Ok(None); + } + let mut variants = Vec::new(); + for value in &schema.enum_values { + variants.push(EnumVariant { + name: type_name(value), + rename: Some(value.clone()), + kind: EnumVariantKind::Unit, + }); + } + Ok(Some(variants)) +} + +fn discriminator_tag(schema: &Schema) -> Option<&str> { + let discriminator = schema.discriminator.as_ref()?; + if !schema.one_of.is_empty() || !discriminator.mapping.is_empty() { + Some(discriminator.property_name.as_str()) + } else { + None + } +} + +fn transparent_all_of(schema: &Schema) -> Option<&MaybeRef> { + let [item] = schema.all_of.as_slice() else { + return None; + }; + (schema.r#type.is_none() + && schema.properties.is_empty() + && schema.required.is_empty() + && schema.items.is_none() + && schema.additional_properties.is_none() + && schema.one_of.is_empty() + && schema.any_of.is_empty() + && schema.enum_values.is_empty() + && schema.const_value.is_none()) + .then_some(item) +} + +fn should_define_inline_schemas(schema: &Schema) -> bool { + transparent_all_of(schema).is_none() + && (!schema.enum_values.is_empty() && has_schema_type(schema, SchemaType::String) + || discriminator_tag(schema).is_some() + || !schema.one_of.is_empty() + || !schema.any_of.is_empty() + || !schema.properties.is_empty() + || schema.all_of.iter().any(|schema| match schema { + MaybeRef::Value(schema) => { + !schema.properties.is_empty() || !schema.required.is_empty() + } + MaybeRef::Ref(_) => false, + })) +} + +pub(super) struct SchemaDefinition { + type_name: RustName, + summary: Option, + description: Option, + module_name: Option, + inline_definitions: Vec, + kind: SchemaKind, +} + +enum SchemaKind { + Enum { + tag: Option, + untagged: bool, + variants: Vec, + }, + Struct { + fields: Vec, + }, + Transparent { + rust_type: RustType, + }, +} + +struct EnumVariant { + name: RustName, + rename: Option, + kind: EnumVariantKind, +} + +enum EnumVariantKind { + Unit, + Tuple { rust_type: RustType }, + Struct { fields: Vec }, +} + +struct SchemaField { + name: String, + identifier: RustName, + rename: Option, + is_required: bool, + rust_type: RustType, +} + +struct InlineSchemas { + module_name: RustName, + definitions: Vec, +} + +impl InlineSchemas { + fn new(name: &str) -> Self { + Self { + module_name: value_name(name), + definitions: Vec::new(), + } + } + + fn module_name(&self) -> Option { + if self.definitions.is_empty() { + return None; + } + Some(self.module_name.clone()) + } +} + +impl SchemaDefinition { + fn new(type_name: RustName, schema: &Schema, kind: SchemaKind) -> Self { + Self { + type_name, + summary: schema.summary.clone(), + description: schema.description.clone(), + module_name: None, + inline_definitions: Vec::new(), + kind, + } + } + + pub(super) fn tokens(&self) -> BuildResult { + let module = if let Some(module_name) = &self.module_name { + let definitions = self + .inline_definitions + .iter() + .map(SchemaDefinition::non_inline_tokens) + .collect::>>()?; + Some(quote! { + pub mod #module_name { + use super::*; + + #(#definitions)* + } + }) + } else { + None + }; + let schema = self.non_inline_tokens()?; + Ok(quote! { + #module + #schema + }) + } + + fn non_inline_tokens(&self) -> BuildResult { + let type_name = &self.type_name; + let docs = doc_attrs(self.summary.as_deref(), self.description.as_deref()); + let derives = quote! { #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] }; + match &self.kind { + SchemaKind::Enum { + tag, + untagged, + variants, + } => { + let serde_tag = tag + .as_ref() + .map(|tag| quote! { #[serde(tag = #tag)] }) + .or_else(|| untagged.then(|| quote! { #[serde(untagged)] })); + let display = (tag.is_none() + && !*untagged + && variants + .iter() + .all(|x| matches!(x.kind, EnumVariantKind::Unit))) + .then(|| generate_enum_display_impl(type_name, variants)); + let variants = variants.iter().map(generate_enum_variant); + Ok(quote! { + #(#docs)* + #derives + #serde_tag + pub enum #type_name { + #(#variants)* + } + + #display + }) + } + SchemaKind::Struct { fields } => { + let fields = fields.iter().map(generate_schema_field); + Ok(quote! { + #(#docs)* + #derives + pub struct #type_name { + #(#fields)* + } + }) + } + SchemaKind::Transparent { rust_type } => { + let display = generate_transparent_display_impl(type_name, rust_type); + Ok(quote! { + #(#docs)* + #derives + #[serde(transparent)] + pub struct #type_name(pub #rust_type); + + #display + }) + } + } + } +} + +fn generate_enum_display_impl(type_name: &RustName, variants: &[EnumVariant]) -> TokenStream { + let arms = variants.iter().map(|variant| { + let name = &variant.name; + let value = variant + .rename + .clone() + .unwrap_or_else(|| variant.name.to_string()); + quote! { + Self::#name => f.write_str(#value), + } + }); + quote! { + impl ::std::fmt::Display for #type_name { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + match self { + #(#arms)* + } + } + } + } +} + +fn generate_transparent_display_impl( + type_name: &RustName, + rust_type: &RustType, +) -> Option { + rust_type_is_displayable(rust_type).then(|| { + quote! { + impl ::std::fmt::Display for #type_name { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + self.0.fmt(f) + } + } + } + }) +} + +fn rust_type_is_displayable(rust_type: &RustType) -> bool { + match rust_type { + RustType::Bool + | RustType::I32 + | RustType::I64 + | RustType::F32 + | RustType::F64 + | RustType::String => true, + RustType::Unit + | RustType::JsonValue + | RustType::Named { .. } + | RustType::Box(_) + | RustType::Option(_) + | RustType::Vec(_) + | RustType::Map(_) => false, + } +} + +fn generate_enum_variant(variant: &EnumVariant) -> TokenStream { + let name = &variant.name; + let rename = variant + .rename + .as_ref() + .map(|value| quote! { #[serde(rename = #value)] }); + match &variant.kind { + EnumVariantKind::Unit => { + quote! { + #rename + #name, + } + } + EnumVariantKind::Tuple { rust_type } => { + quote! { + #rename + #name(#rust_type), + } + } + EnumVariantKind::Struct { fields } => { + let fields = fields.iter().map(generate_enum_variant_field); + quote! { + #rename + #name { + #(#fields)* + }, + } + } + } +} + +fn generate_enum_variant_field(field: &SchemaField) -> TokenStream { + generate_field(field, quote! {}) +} + +fn generate_schema_field(field: &SchemaField) -> TokenStream { + generate_field(field, quote! { pub }) +} + +fn generate_field(field: &SchemaField, visibility: TokenStream) -> TokenStream { + let name = &field.identifier; + let rust_type = &field.rust_type; + let rename = field + .rename + .as_ref() + .map(|value| quote! { #[serde(rename = #value)] }); + let optional = (!field.is_required && field.rust_type.is_option()) + .then(|| quote! { #[serde(default, skip_serializing_if = "Option::is_none")] }); + quote! { + #rename + #optional + #visibility #name: #rust_type, + } +} diff --git a/crates/sail-build-scripts/src/openapi/mod.rs b/crates/sail-build-scripts/src/openapi/mod.rs new file mode 100644 index 000000000..d6a1fb894 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/mod.rs @@ -0,0 +1,3 @@ +pub mod generator; +pub mod spec; +mod utils; diff --git a/crates/sail-build-scripts/src/openapi/spec.rs b/crates/sail-build-scripts/src/openapi/spec.rs new file mode 100644 index 000000000..d0bca8204 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/spec.rs @@ -0,0 +1,451 @@ +//! Rust types for the OpenAPI specification. +//! +//! The types are compatible with a subset of OpenAPI 3.0 and 3.1 (which has a few breaking changes +//! between them). The types are designed to support client code generation for REST services +//! used in the codebase (e.g., Unity Catalog and Iceberg REST catalog). +//! The types can be extended if more OpenAPI features are needed in the future. +use std::collections::BTreeMap; +use std::fmt; +use std::path::Path; + +use serde::Deserialize; +use serde::de::{self, IgnoredAny, Visitor}; + +use crate::error::BuildResult; + +pub fn load_spec(path: impl AsRef) -> BuildResult { + let content = std::fs::read_to_string(path)?; + Ok(serde_yaml::from_str(&content)?) +} + +pub type ObjectMap = BTreeMap; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OpenApi { + pub openapi: String, + pub info: Info, + #[serde(default)] + pub servers: Vec, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub paths: ObjectMap>, + #[serde(default)] + pub components: Components, + #[serde(default)] + pub security: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Info { + pub title: String, + pub version: String, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub license: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct License { + pub name: String, + #[serde(default)] + pub url: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Server { + pub url: String, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub variables: ObjectMap, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ServerVariable { + pub default: String, + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Tag { + pub name: String, + #[serde(default)] + pub description: Option, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Components { + #[serde(default)] + pub schemas: ObjectMap>, + #[serde(default)] + pub parameters: ObjectMap>, + #[serde(default)] + pub responses: ObjectMap>, + #[serde(default)] + pub examples: ObjectMap, + #[serde(default, rename = "securitySchemes")] + pub security_schemes: ObjectMap, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PathItem { + #[serde(default)] + pub summary: Option, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub parameters: Vec>, + #[serde(default)] + pub get: Option, + #[serde(default)] + pub put: Option, + #[serde(default)] + pub post: Option, + #[serde(default)] + pub delete: Option, + #[serde(default)] + pub options: Option, + #[serde(default)] + pub head: Option, + #[serde(default)] + pub patch: Option, + #[serde(default)] + pub trace: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Operation { + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub summary: Option, + #[serde(default)] + pub description: Option, + #[serde(default, rename = "operationId")] + pub operation_id: Option, + #[serde(default)] + pub parameters: Vec>, + #[serde(default, rename = "requestBody")] + pub request_body: Option>, + #[serde(default, deserialize_with = "deserialize_string_key_map")] + pub responses: ObjectMap>, + #[serde(default)] + pub deprecated: Option, + #[serde(default)] + pub security: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Parameter { + pub name: String, + #[serde(rename = "in")] + pub location: ParameterLocation, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub required: Option, + #[serde(default)] + pub schema: Option>, + #[serde(default)] + pub style: Option, + #[serde(default)] + pub explode: Option, + #[serde(default, rename = "allowEmptyValue")] + pub allow_empty_value: Option, + #[serde(default)] + pub example: Option, + #[serde(default)] + pub examples: ObjectMap, +} + +#[derive(Debug, Clone, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ParameterLocation { + Query, + Header, + Path, + Cookie, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RequestBody { + #[serde(default)] + pub description: Option, + #[serde(default)] + pub content: ObjectMap, + #[serde(default)] + pub required: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Response { + pub description: String, + #[serde(default)] + pub content: ObjectMap, + #[serde(default)] + pub headers: ObjectMap>, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Header { + #[serde(default)] + pub description: Option, + #[serde(default)] + pub required: Option, + #[serde(default)] + pub deprecated: Option, + #[serde(default, rename = "allowEmptyValue")] + pub allow_empty_value: Option, + #[serde(default)] + pub style: Option, + #[serde(default)] + pub explode: Option, + #[serde(default)] + pub schema: Option>, + #[serde(default)] + pub example: Option, + #[serde(default)] + pub examples: ObjectMap, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MediaType { + #[serde(default)] + pub schema: Option>, + #[serde(default)] + pub example: Option, + #[serde(default)] + pub examples: ObjectMap, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Schema { + #[serde( + default, + rename = "type", + deserialize_with = "deserialize_schema_types" + )] + pub r#type: Option>, + #[serde(default)] + pub format: Option, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub summary: Option, + #[serde(default)] + pub default: Option, + #[serde(default)] + pub example: Option, + #[serde(default, rename = "enum")] + pub enum_values: Vec, + #[serde(default, rename = "const")] + pub const_value: Option, + #[serde(default)] + pub properties: ObjectMap>, + #[serde(default)] + pub required: Vec, + #[serde(default)] + pub items: Option>>, + #[serde(default, rename = "additionalProperties")] + pub additional_properties: Option, + #[serde(default, rename = "allOf")] + pub all_of: Vec>, + #[serde(default, rename = "oneOf")] + pub one_of: Vec>, + #[serde(default, rename = "anyOf")] + pub any_of: Vec>, + #[serde(default)] + pub nullable: Option, + #[serde(default)] + pub discriminator: Option, + #[serde(default)] + pub deprecated: Option, + #[serde(default, rename = "readOnly")] + pub read_only: Option, + #[serde(default)] + pub minimum: Option, + #[serde(default)] + pub maximum: Option, + #[serde(default, rename = "minLength")] + pub min_length: Option, + #[serde(default, rename = "maxLength")] + pub max_length: Option, + #[serde(default, rename = "maxItems")] + pub max_items: Option, + #[serde(default, rename = "uniqueItems")] + pub unique_items: Option, + #[serde(default)] + pub pattern: Option, + #[serde(default, rename = "contentEncoding")] + pub content_encoding: Option, + #[serde(default, rename = "x-enum-descriptions")] + pub x_enum_descriptions: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum AdditionalProperties { + Bool(bool), + Schema(Box>), +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Discriminator { + #[serde(rename = "propertyName")] + pub property_name: String, + #[serde(default)] + pub mapping: ObjectMap, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SchemaType { + Null, + Boolean, + Object, + Array, + Number, + String, + Integer, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum MaybeRef { + Ref(R), + Value(T), +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Reference { + #[serde(rename = "$ref")] + pub reference: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SchemaReference { + #[serde(rename = "$ref")] + pub reference: String, + #[serde(default)] + pub description: Option, + #[serde(default, rename = "const")] + pub const_value: Option, + #[serde(default, rename = "enum")] + pub enum_values: Vec, +} + +/// A placeholder type for any value that we do not care about during deserialization. +#[derive(Debug, Clone)] +pub struct AnyValue; + +impl<'de> Deserialize<'de> for AnyValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + IgnoredAny::deserialize(deserializer)?; + Ok(Self) + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum OneOrMany { + One(T), + Many(Vec), +} + +fn deserialize_schema_types<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + Ok( + Option::>::deserialize(deserializer)?.map(|value| match value { + OneOrMany::One(value) => vec![value], + OneOrMany::Many(value) => value, + }), + ) +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] +struct StringKey(String); + +impl<'de> Deserialize<'de> for StringKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(StringKeyVisitor) + } +} + +struct StringKeyVisitor; + +impl Visitor<'_> for StringKeyVisitor { + type Value = StringKey; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string or integer map key") + } + + fn visit_i64(self, value: i64) -> Result + where + E: de::Error, + { + Ok(StringKey(value.to_string())) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + Ok(StringKey(value.to_string())) + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(StringKey(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result + where + E: de::Error, + { + Ok(StringKey(value)) + } +} + +fn deserialize_string_key_map<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + let values = BTreeMap::::deserialize(deserializer)?; + Ok(values + .into_iter() + .map(|(StringKey(key), value)| (key, value)) + .collect()) +} diff --git a/crates/sail-build-scripts/src/openapi/utils/docs.rs b/crates/sail-build-scripts/src/openapi/utils/docs.rs new file mode 100644 index 000000000..69045e369 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/utils/docs.rs @@ -0,0 +1,24 @@ +use proc_macro2::TokenStream; +use quote::quote; + +pub fn doc_attrs(summary: Option<&str>, description: Option<&str>) -> Vec { + let mut lines = Vec::new(); + if let Some(summary) = summary { + lines.extend(doc_lines(summary)); + } + if summary.is_some() && description.is_some() { + lines.push(String::new()); + } + if let Some(description) = description { + lines.extend(doc_lines(description)); + } + lines + .into_iter() + .map(|line| format!(" {line}")) + .map(|line| quote! { #[doc = #line] }) + .collect() +} + +fn doc_lines(value: &str) -> impl Iterator + '_ { + value.lines().map(|line| line.replace('\r', "")) +} diff --git a/crates/sail-build-scripts/src/openapi/utils/http.rs b/crates/sail-build-scripts/src/openapi/utils/http.rs new file mode 100644 index 000000000..5e03ac911 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/utils/http.rs @@ -0,0 +1,128 @@ +use crate::error::{BuildError, BuildResult}; + +#[derive(Debug, Clone, Copy)] +pub enum HttpMethod { + Get, + Put, + Post, + Delete, + Options, + Head, + Patch, + Trace, +} + +impl HttpMethod { + pub fn name(self) -> &'static str { + match self { + Self::Get => "GET", + Self::Put => "PUT", + Self::Post => "POST", + Self::Delete => "DELETE", + Self::Options => "OPTIONS", + Self::Head => "HEAD", + Self::Patch => "PATCH", + Self::Trace => "TRACE", + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum HttpStatus { + Success(u16), + ExactError(u16), + ClientError, + ServerError, +} + +impl HttpStatus { + pub fn parse(status: &str) -> BuildResult { + if status == "4XX" { + return Ok(Self::ClientError); + } + if status == "5XX" { + return Ok(Self::ServerError); + } + if let Ok(status) = status.parse::() { + if (200..300).contains(&status) { + return Ok(Self::Success(status)); + } + if (300..600).contains(&status) { + return Ok(Self::ExactError(status)); + } + } + Err(BuildError::InvalidInput(format!( + "unsupported response status: {status}" + ))) + } + + pub fn range(self) -> Option<(u16, u16)> { + match self { + Self::ClientError => Some((400, 499)), + Self::ServerError => Some((500, 599)), + Self::Success(_) | Self::ExactError(_) => None, + } + } + + pub fn variant(self) -> BuildResult<&'static str> { + Ok(match self { + Self::Success(_) => "Success", + Self::ClientError => "ClientError", + Self::ServerError => "ServerError", + Self::ExactError(status) => match status { + 300 => "MultipleChoices", + 301 => "MovedPermanently", + 302 => "Found", + 303 => "SeeOther", + 304 => "NotModified", + 307 => "TemporaryRedirect", + 308 => "PermanentRedirect", + 400 => "BadRequest", + 401 => "Unauthorized", + 402 => "PaymentRequired", + 403 => "Forbidden", + 404 => "NotFound", + 405 => "MethodNotAllowed", + 406 => "NotAcceptable", + 407 => "ProxyAuthenticationRequired", + 408 => "RequestTimeout", + 409 => "Conflict", + 410 => "Gone", + 411 => "LengthRequired", + 412 => "PreconditionFailed", + 413 => "PayloadTooLarge", + 414 => "UriTooLong", + 415 => "UnsupportedMediaType", + 416 => "RangeNotSatisfiable", + 417 => "ExpectationFailed", + 418 => "ImATeapot", + 419 => "AuthenticationTimeout", + 422 => "UnprocessableEntity", + 423 => "Locked", + 424 => "FailedDependency", + 425 => "TooEarly", + 426 => "UpgradeRequired", + 428 => "PreconditionRequired", + 429 => "TooManyRequests", + 431 => "RequestHeaderFieldsTooLarge", + 451 => "UnavailableForLegalReasons", + 500 => "InternalServerError", + 501 => "NotImplemented", + 502 => "BadGateway", + 503 => "ServiceUnavailable", + 504 => "GatewayTimeout", + 505 => "HttpVersionNotSupported", + 506 => "VariantAlsoNegotiates", + 507 => "InsufficientStorage", + 508 => "LoopDetected", + 510 => "NotExtended", + 511 => "NetworkAuthenticationRequired", + _ => { + return Err(BuildError::InvalidInput(format!( + "unsupported response status: {status}" + ))); + } + }, + }) + } +} diff --git a/crates/sail-build-scripts/src/openapi/utils/mod.rs b/crates/sail-build-scripts/src/openapi/utils/mod.rs new file mode 100644 index 000000000..8cba87c94 --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/utils/mod.rs @@ -0,0 +1,4 @@ +pub(super) mod docs; +pub(super) mod http; +pub(super) mod name; +pub(super) mod types; diff --git a/crates/sail-build-scripts/src/openapi/utils/name.rs b/crates/sail-build-scripts/src/openapi/utils/name.rs new file mode 100644 index 000000000..9b16ef15d --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/utils/name.rs @@ -0,0 +1,101 @@ +use std::fmt::{Display, Formatter}; + +use proc_macro2::{Ident, TokenStream}; +use quote::{TokenStreamExt, format_ident}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RustName(String); + +impl RustName { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } +} + +impl Display for RustName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl quote::ToTokens for RustName { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(format_ident!("{}", self.0)); + } +} + +pub fn lowercase_parts(value: &str) -> Vec { + let mut output = Vec::new(); + let mut part = String::new(); + let characters = value.chars().collect::>(); + for (index, character) in characters.iter().copied().enumerate() { + if character.is_ascii_uppercase() { + let previous = index.checked_sub(1).and_then(|index| characters.get(index)); + let next = characters.get(index + 1); + if previous.is_some_and(|character| { + character.is_ascii_lowercase() + || character.is_ascii_digit() + || character.is_ascii_uppercase() + && next.is_some_and(|character| character.is_ascii_lowercase()) + }) && !part.is_empty() + { + output.push(std::mem::take(&mut part)); + } + part.push(character.to_ascii_lowercase()); + } else if character.is_ascii_alphanumeric() { + part.push(character.to_ascii_lowercase()); + } else if !part.is_empty() { + output.push(std::mem::take(&mut part)); + } + } + if !part.is_empty() { + output.push(part); + } + output +} + +pub fn type_name(value: &str) -> RustName { + let mut output = String::new(); + for part in lowercase_parts(value) { + let mut characters = part.chars(); + if let Some(first) = characters.next() { + output.push(first.to_ascii_uppercase()); + for character in characters { + output.push(character.to_ascii_lowercase()); + } + } + } + RustName::new(if output.is_empty() { + "Value".to_owned() + } else if output + .chars() + .next() + .is_some_and(|character| character.is_ascii_digit()) + { + format!("Value{output}") + } else { + output + }) +} + +pub fn value_name(value: &str) -> RustName { + let value = lowercase_parts(value).join("_"); + let value = if value.is_empty() { + "value".to_owned() + } else if value + .chars() + .next() + .is_some_and(|character| character.is_ascii_digit()) + { + format!("_{value}") + } else { + value + }; + if syn::parse_str::(&value).is_ok() { + RustName::new(value) + } else if syn::parse_str::(&format!("r#{value}")).is_ok() { + RustName::new(format!("r#{value}")) + } else { + RustName::new(format!("{value}_")) + } +} diff --git a/crates/sail-build-scripts/src/openapi/utils/types.rs b/crates/sail-build-scripts/src/openapi/utils/types.rs new file mode 100644 index 000000000..c0be9d15f --- /dev/null +++ b/crates/sail-build-scripts/src/openapi/utils/types.rs @@ -0,0 +1,79 @@ +use proc_macro2::TokenStream; +use quote::quote; + +use crate::openapi::utils::name::RustName; + +#[derive(Clone)] +pub enum RustType { + Unit, + Bool, + I32, + I64, + F32, + F64, + String, + JsonValue, + Named { + qualifier: Vec, + name: RustName, + }, + Box(Box), + Option(Box), + Vec(Box), + Map(Box), +} + +impl RustType { + pub fn is_unit(&self) -> bool { + matches!(self, Self::Unit) + } + + pub fn is_string(&self) -> bool { + match self { + Self::String => true, + Self::Option(value) | Self::Box(value) => value.is_string(), + _ => false, + } + } + + pub fn is_vec(&self) -> bool { + matches!(self, Self::Vec(_)) + } + + pub fn is_option(&self) -> bool { + matches!(self, Self::Option(_)) + } +} + +#[derive(Clone, Copy)] +pub(crate) enum TypePosition { + Normal, + Nested, +} + +impl quote::ToTokens for RustType { + fn to_tokens(&self, tokens: &mut TokenStream) { + let value = match self { + Self::Unit => quote! { () }, + Self::Bool => quote! { bool }, + Self::I32 => quote! { i32 }, + Self::I64 => quote! { i64 }, + Self::F32 => quote! { f32 }, + Self::F64 => quote! { f64 }, + Self::String => quote! { String }, + Self::JsonValue => quote! { serde_json::Value }, + Self::Named { qualifier, name } => { + let segments = qualifier + .iter() + .chain(std::iter::once(name)) + .collect::>(); + quote! { #(#segments)::* } + } + Self::Box(value) => quote! { Box<#value> }, + Self::Option(value) => quote! { Option<#value> }, + Self::Vec(value) => quote! { Vec<#value> }, + Self::Map(value) => quote! { std::collections::HashMap }, + }; + tokens.extend(value); + } +} diff --git a/crates/sail-cache/Cargo.toml b/crates/sail-cache/Cargo.toml index cd6b240ee..db7475d10 100644 --- a/crates/sail-cache/Cargo.toml +++ b/crates/sail-cache/Cargo.toml @@ -8,10 +8,13 @@ workspace = true [dependencies] sail-common = { path = "../sail-common" } +sail-common-datafusion = { path = "../sail-common-datafusion" } +sail-object-store = { path = "../sail-object-store" } thiserror = { workspace = true } object_store = { workspace = true } datafusion = { workspace = true } +datafusion-common = { workspace = true } chrono = { workspace = true } log = { workspace = true } moka = { workspace = true } diff --git a/crates/sail-cache/src/file_metadata_cache.rs b/crates/sail-cache/src/file_metadata_cache.rs index a1059b01b..37292fccb 100644 --- a/crates/sail-cache/src/file_metadata_cache.rs +++ b/crates/sail-cache/src/file_metadata_cache.rs @@ -1,10 +1,10 @@ use std::collections::HashMap; use std::time::Duration; +use datafusion::execution::cache::CacheAccessor; use datafusion::execution::cache::cache_manager::{ CachedFileMetadataEntry, FileMetadataCache, FileMetadataCacheEntry, }; -use datafusion::execution::cache::CacheAccessor; use log::debug; use moka::policy::EvictionPolicy; use moka::sync::Cache; @@ -116,8 +116,8 @@ mod tests { use chrono::DateTime; use datafusion::execution::cache::cache_manager::FileMetadata; - use object_store::path::Path; use object_store::ObjectMeta; + use object_store::path::Path; use super::*; diff --git a/crates/sail-cache/src/file_statistics_cache.rs b/crates/sail-cache/src/file_statistics_cache.rs index e9cc8bbe8..6246997c0 100644 --- a/crates/sail-cache/src/file_statistics_cache.rs +++ b/crates/sail-cache/src/file_statistics_cache.rs @@ -123,8 +123,8 @@ mod tests { use chrono::DateTime; use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datafusion::common::Statistics; - use object_store::path::Path; use object_store::ObjectMeta; + use object_store::path::Path; use super::*; @@ -162,10 +162,12 @@ mod tests { let mut meta2 = meta.clone(); meta2.size = 2048; let key2 = scoped_path(meta2.location.clone()); - assert!(!cache - .get(&key2) - .map(|c| c.is_valid_for(&meta2)) - .unwrap_or(false)); + assert!( + !cache + .get(&key2) + .map(|c| c.is_valid_for(&meta2)) + .unwrap_or(false) + ); // different file let mut meta2 = meta; diff --git a/crates/sail-cache/src/lib.rs b/crates/sail-cache/src/lib.rs index a8fe1223a..422e2fb1a 100644 --- a/crates/sail-cache/src/lib.rs +++ b/crates/sail-cache/src/lib.rs @@ -4,6 +4,7 @@ pub mod error; pub mod file_listing_cache; pub mod file_metadata_cache; pub mod file_statistics_cache; +pub mod remote_checkpoint; #[expect(dead_code)] pub(crate) fn try_parse_memory_limit(limit: &str) -> Option { diff --git a/crates/sail-cache/src/remote_checkpoint.rs b/crates/sail-cache/src/remote_checkpoint.rs new file mode 100644 index 000000000..2fd9d156b --- /dev/null +++ b/crates/sail-cache/src/remote_checkpoint.rs @@ -0,0 +1,225 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock}; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::physical_expr::{LexOrdering, Partitioning}; +use datafusion_common::{DataFusionError, Result, internal_datafusion_err}; +use object_store::path::Path; +use sail_common_datafusion::extension::SessionExtension; +use sail_object_store::{ + ResolvedObjectStorePath, delete_object_store_prefix_objects, resolve_object_store_path, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RemoteCheckpointFile { + pub location: Path, + pub size: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RemoteCheckpointPartition { + pub partition: usize, + pub row_count: u64, + pub file: Option, +} + +#[derive(Clone, Debug)] +pub struct RemoteCheckpointDescriptor { + pub relation_id: String, + pub object_store_url: ObjectStoreUrl, + pub prefix: Path, + pub logical_schema: SchemaRef, + pub storage_schema: SchemaRef, + pub output_partitioning: Partitioning, + pub output_ordering: Option, + pub partitions: Vec, +} + +/// Session-local publication index for complete checkpoints; object-store contents are not +/// reloaded after a process restart. +#[derive(Debug)] +pub struct RemoteCheckpointRegistry { + path: Option, + session_id: String, + closed: AtomicBool, + relations: RwLock>>, +} + +impl RemoteCheckpointRegistry { + pub fn new(path: Option, session_id: String) -> Self { + Self { + path, + session_id, + closed: AtomicBool::new(false), + relations: RwLock::new(HashMap::new()), + } + } + + pub fn resolve_relation( + &self, + runtime_env: &RuntimeEnv, + relation_id: &str, + ) -> Result<(ObjectStoreUrl, Path)> { + if self.closed.load(Ordering::Acquire) { + return Err(DataFusionError::Plan( + "checkpoint session is stopping".to_string(), + )); + } + let root = self.resolve_root(runtime_env)?; + let prefix = self.session_prefix(root.prefix()).join(relation_id); + Ok((root.object_store_url().clone(), prefix)) + } + + pub fn insert(&self, descriptor: RemoteCheckpointDescriptor) -> Result<()> { + if self.closed.load(Ordering::Acquire) { + return Err(DataFusionError::Plan( + "checkpoint session is stopping".to_string(), + )); + } + let relation_id = descriptor.relation_id.clone(); + let mut relations = self.relations.write().map_err(|error| { + internal_datafusion_err!("checkpoint registry is poisoned: {error}") + })?; + match relations.entry(relation_id.clone()) { + Entry::Occupied(_) => { + return Err(internal_datafusion_err!( + "checkpoint relation already exists: {relation_id}" + )); + } + Entry::Vacant(entry) => { + entry.insert(Arc::new(descriptor)); + } + } + Ok(()) + } + + pub fn get(&self, relation_id: &str) -> Result>> { + let relations = self.relations.read().map_err(|error| { + internal_datafusion_err!("checkpoint registry is poisoned: {error}") + })?; + Ok(relations.get(relation_id).cloned()) + } + + pub async fn cleanup_relation( + &self, + runtime_env: &RuntimeEnv, + object_store_url: &ObjectStoreUrl, + prefix: &Path, + ) -> Result<()> { + let store = runtime_env.object_store(object_store_url)?; + delete_object_store_prefix_objects(store.as_ref(), prefix).await + } + + pub async fn cleanup_session(&self, runtime_env: &RuntimeEnv) -> Result<()> { + self.closed.store(true, Ordering::Release); + let storage_cleanup = if self.path.is_some() { + match self.resolve_root(runtime_env) { + Ok(root) => { + delete_object_store_prefix_objects( + root.store().as_ref(), + &self.session_prefix(root.prefix()), + ) + .await + } + Err(error) => Err(error), + } + } else { + Ok(()) + }; + let registry_cleanup = self.clear(); + match (storage_cleanup, registry_cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(storage_error), Err(registry_error)) => Err(internal_datafusion_err!( + "checkpoint storage cleanup failed: {storage_error}; registry cleanup failed: {registry_error}" + )), + } + } + + fn resolve_root(&self, runtime_env: &RuntimeEnv) -> Result { + let path = self.path.as_deref().ok_or_else(|| { + DataFusionError::Plan( + "checkpoint object-store path is not configured; set execution.checkpoint.path" + .to_string(), + ) + })?; + resolve_object_store_path(runtime_env, path) + } + + fn session_prefix(&self, root: &Path) -> Path { + root.clone().join(self.session_id.as_str()) + } + + fn clear(&self) -> Result<()> { + let mut relations = self.relations.write().map_err(|error| { + internal_datafusion_err!("checkpoint registry is poisoned: {error}") + })?; + relations.clear(); + Ok(()) + } +} + +impl SessionExtension for RemoteCheckpointRegistry { + fn name() -> &'static str { + "remote checkpoint registry" + } +} + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use datafusion::arrow::datatypes::Schema; + + use super::*; + + fn descriptor(relation_id: &str) -> RemoteCheckpointDescriptor { + let schema = Arc::new(Schema::empty()); + RemoteCheckpointDescriptor { + relation_id: relation_id.to_string(), + object_store_url: ObjectStoreUrl::parse("memory://").unwrap(), + prefix: Path::from("checkpoint"), + logical_schema: Arc::clone(&schema), + storage_schema: schema, + output_partitioning: Partitioning::UnknownPartitioning(1), + output_ordering: None, + partitions: vec![], + } + } + + #[test] + fn registry_owns_only_ready_descriptors() -> Result<()> { + let registry = RemoteCheckpointRegistry::new(None, "session".to_string()); + registry.insert(descriptor("relation"))?; + assert_eq!( + registry + .get("relation")? + .expect("descriptor must be present") + .relation_id, + "relation" + ); + let original = registry + .get("relation")? + .expect("descriptor must be present"); + assert!(registry.insert(descriptor("relation")).is_err()); + let retained = registry + .get("relation")? + .expect("descriptor must remain present"); + assert!(Arc::ptr_eq(&original, &retained)); + registry.clear()?; + assert!(registry.get("relation")?.is_none()); + Ok(()) + } + + #[test] + fn registry_uses_the_provided_session_id_for_storage_paths() { + let registry = RemoteCheckpointRegistry::new(None, "session-1".to_string()); + assert_eq!( + registry.session_prefix(&Path::from("checkpoint")), + Path::from("checkpoint/session-1") + ); + } +} diff --git a/crates/sail-catalog-glue/Cargo.toml b/crates/sail-catalog-glue/Cargo.toml index 58446ae98..193794d01 100644 --- a/crates/sail-catalog-glue/Cargo.toml +++ b/crates/sail-catalog-glue/Cargo.toml @@ -13,6 +13,7 @@ sail-catalog = { path = "../sail-catalog" } arrow = { workspace = true } async-trait = { workspace = true } aws-config = { workspace = true } +aws-credential-types = { workspace = true } aws-sdk-glue = { workspace = true } aws-smithy-types = { workspace = true } tokio = { workspace = true } diff --git a/crates/sail-catalog-glue/src/data_type.rs b/crates/sail-catalog-glue/src/data_type.rs index b730f9964..386352312 100644 --- a/crates/sail-catalog-glue/src/data_type.rs +++ b/crates/sail-catalog-glue/src/data_type.rs @@ -51,12 +51,12 @@ pub fn arrow_to_glue_type(data_type: &DataType) -> CatalogResult { Ok(format!("struct<{}>", field_strs?.join(","))) } DataType::Map(field, _) => { - if let DataType::Struct(fields) = field.data_type() { - if fields.len() == 2 { - let key_type = arrow_to_glue_type(fields[0].data_type())?; - let value_type = arrow_to_glue_type(fields[1].data_type())?; - return Ok(format!("map<{key_type},{value_type}>")); - } + if let DataType::Struct(fields) = field.data_type() + && fields.len() == 2 + { + let key_type = arrow_to_glue_type(fields[0].data_type())?; + let value_type = arrow_to_glue_type(fields[1].data_type())?; + return Ok(format!("map<{key_type},{value_type}>")); } Err(CatalogError::InvalidArgument( "Map type must have key and value fields".to_string(), @@ -162,32 +162,32 @@ pub fn arrow_to_iceberg_type(data_type: &DataType) -> CatalogResult { )) } DataType::Map(field, _) => { - if let DataType::Struct(fields) = field.data_type() { - if fields.len() == 2 { - let key_type = arrow_to_iceberg_type(fields[0].data_type())?; - let value_type = arrow_to_iceberg_type(fields[1].data_type())?; - return Ok(Document::Object( - vec![ - ("type".to_string(), Document::String("map".to_string())), - ( - "key-id".to_string(), - Document::Number(aws_smithy_types::Number::PosInt(1)), - ), - ("key".to_string(), key_type), - ( - "value-id".to_string(), - Document::Number(aws_smithy_types::Number::PosInt(2)), - ), - ("value".to_string(), value_type), - ( - "value-required".to_string(), - Document::Bool(!fields[1].is_nullable()), - ), - ] - .into_iter() - .collect(), - )); - } + if let DataType::Struct(fields) = field.data_type() + && fields.len() == 2 + { + let key_type = arrow_to_iceberg_type(fields[0].data_type())?; + let value_type = arrow_to_iceberg_type(fields[1].data_type())?; + return Ok(Document::Object( + vec![ + ("type".to_string(), Document::String("map".to_string())), + ( + "key-id".to_string(), + Document::Number(aws_smithy_types::Number::PosInt(1)), + ), + ("key".to_string(), key_type), + ( + "value-id".to_string(), + Document::Number(aws_smithy_types::Number::PosInt(2)), + ), + ("value".to_string(), value_type), + ( + "value-required".to_string(), + Document::Bool(!fields[1].is_nullable()), + ), + ] + .into_iter() + .collect(), + )); } Err(CatalogError::InvalidArgument( "Map type must have key and value fields".to_string(), @@ -455,12 +455,11 @@ mod tests { let map_type = DataType::Map(Arc::new(map_field), false); let glue_str = arrow_to_glue_type(&map_type).unwrap(); let roundtrip = glue_type_to_arrow(&glue_str).unwrap(); - if let (DataType::Map(orig, _), DataType::Map(rt, _)) = (&map_type, &roundtrip) { - if let (DataType::Struct(of), DataType::Struct(rf)) = (orig.data_type(), rt.data_type()) - { - assert_eq!(of[0].data_type(), rf[0].data_type()); - assert_eq!(of[1].data_type(), rf[1].data_type()); - } + if let (DataType::Map(orig, _), DataType::Map(rt, _)) = (&map_type, &roundtrip) + && let (DataType::Struct(of), DataType::Struct(rf)) = (orig.data_type(), rt.data_type()) + { + assert_eq!(of[0].data_type(), rf[0].data_type()); + assert_eq!(of[1].data_type(), rf[1].data_type()); } let nested = DataType::List(Arc::new(Field::new( diff --git a/crates/sail-catalog-glue/src/hive.rs b/crates/sail-catalog-glue/src/hive.rs index c126555b8..8ffc30e2d 100644 --- a/crates/sail-catalog-glue/src/hive.rs +++ b/crates/sail-catalog-glue/src/hive.rs @@ -1,8 +1,8 @@ /// Hive-style table creation logic for AWS Glue Data Catalog. use std::collections::{HashMap, HashSet}; -use aws_sdk_glue::types::{SerDeInfo, StorageDescriptor, TableInput}; use aws_sdk_glue::Client; +use aws_sdk_glue::types::{SerDeInfo, StorageDescriptor, TableInput}; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::hive_format::{HiveCatalogFormat, HiveStorageFormat}; use sail_catalog::provider::{ @@ -10,8 +10,8 @@ use sail_catalog::provider::{ }; use sail_common_datafusion::catalog::TableStatus; -use crate::data_type::arrow_to_glue_type; use crate::GlueCatalogProvider; +use crate::data_type::arrow_to_glue_type; pub(crate) const SPARK_DATASOURCE_PROVIDER_KEY: &str = "spark.sql.sources.provider"; @@ -67,6 +67,7 @@ pub(crate) async fn create_hive_table( let result = client .create_table() + .set_catalog_id(provider.catalog_id()) .database_name(&database_name) .table_input(table_input) .send() diff --git a/crates/sail-catalog-glue/src/iceberg.rs b/crates/sail-catalog-glue/src/iceberg.rs index d1097a4e6..53866e601 100644 --- a/crates/sail-catalog-glue/src/iceberg.rs +++ b/crates/sail-catalog-glue/src/iceberg.rs @@ -1,26 +1,26 @@ /// Iceberg table creation logic for AWS Glue Data Catalog. use std::collections::{HashMap, HashSet}; +use aws_sdk_glue::Client; use aws_sdk_glue::types::{ CreateIcebergTableInput, IcebergInput, IcebergPartitionField, IcebergPartitionSpec, IcebergSchema, IcebergStructField, IcebergStructTypeEnum, MetadataOperation, OpenTableFormatInput, SerDeInfo, StorageDescriptor, TableInput, }; -use aws_sdk_glue::Client; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::hive_format::HiveStorageFormat; use sail_catalog::provider::{ CatalogPartitionField, CatalogProvider, CreateTableColumnOptions, CreateTableOptions, Namespace, PartitionTransform, }; +use sail_common_datafusion::catalog::TableStatus; use sail_common_datafusion::catalog::iceberg::{ - is_iceberg_table_properties, ICEBERG_CLASSIFICATION_KEY, ICEBERG_TABLE_TYPE_KEY, - ICEBERG_TABLE_TYPE_VALUE, + ICEBERG_CLASSIFICATION_KEY, ICEBERG_TABLE_TYPE_KEY, ICEBERG_TABLE_TYPE_VALUE, + is_iceberg_table_properties, }; -use sail_common_datafusion::catalog::TableStatus; -use crate::data_type::{arrow_to_glue_type, arrow_to_iceberg_type}; use crate::GlueCatalogProvider; +use crate::data_type::{arrow_to_glue_type, arrow_to_iceberg_type}; /// Default Iceberg table spec version for new tables (v2 is the modern standard). const DEFAULT_ICEBERG_FORMAT_VERSION: &str = "2"; @@ -97,6 +97,7 @@ pub(crate) async fn create_iceberg_table( let result = client .create_table() + .set_catalog_id(provider.catalog_id()) .database_name(&database_name) .name(table) .open_table_format_input(open_format_input) @@ -198,6 +199,7 @@ async fn create_iceberg_table_via_table_input( let result = client .create_table() + .set_catalog_id(provider.catalog_id()) .database_name(&database_name) .table_input(table_input) .send() diff --git a/crates/sail-catalog-glue/src/provider.rs b/crates/sail-catalog-glue/src/provider.rs index e219e4dc3..603295be9 100644 --- a/crates/sail-catalog-glue/src/provider.rs +++ b/crates/sail-catalog-glue/src/provider.rs @@ -1,11 +1,12 @@ use std::collections::HashMap; use aws_config::BehaviorVersion; +use aws_credential_types::Credentials; +use aws_sdk_glue::Client; use aws_sdk_glue::config::Region; use aws_sdk_glue::types::{ StorageDescriptor, Table, TableInput, ViewDefinitionInput, ViewRepresentationInput, }; -use aws_sdk_glue::Client; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::hive_format::HiveDetectedFormat; use sail_catalog::provider::{ @@ -15,7 +16,7 @@ use sail_catalog::provider::{ }; use sail_catalog::utils::quote_namespace_if_needed; use sail_common_datafusion::catalog::{ - identity_partition_fields, DatabaseStatus, TableColumnStatus, TableKind, TableStatus, + DatabaseStatus, TableColumnStatus, TableKind, TableStatus, identity_partition_fields, }; use tokio::sync::OnceCell; @@ -25,6 +26,8 @@ use crate::{hive, iceberg, managed_table}; /// Configuration for AWS Glue Data Catalog. #[derive(Debug, Clone, Default)] pub struct GlueCatalogConfig { + /// AWS Glue Data Catalog ID. If not set, AWS uses the caller's account ID. + pub catalog_id: Option, /// AWS region (e.g., "us-east-1"). If not set, uses default from credential chain. pub region: Option, /// Custom endpoint URL (optional). Useful for VPC endpoints or local development. @@ -35,6 +38,7 @@ pub struct GlueCatalogConfig { pub struct GlueCatalogProvider { name: String, config: GlueCatalogConfig, + credentials: Option, client: OnceCell, } @@ -43,6 +47,29 @@ impl GlueCatalogProvider { Self { name, config, + credentials: None, + client: OnceCell::new(), + } + } + + #[doc(hidden)] + pub fn new_with_static_credentials( + name: String, + config: GlueCatalogConfig, + access_key_id: String, + secret_access_key: String, + session_token: Option, + ) -> Self { + Self { + name, + config, + credentials: Some(Credentials::new( + access_key_id, + secret_access_key, + session_token, + None, + "sail-glue-catalog-config", + )), client: OnceCell::new(), } } @@ -60,6 +87,10 @@ impl GlueCatalogProvider { config_loader = config_loader.endpoint_url(endpoint); } + if let Some(credentials) = &self.credentials { + config_loader = config_loader.credentials_provider(credentials.clone()); + } + let sdk_config = config_loader.load().await; Ok(Client::new(&sdk_config)) }) @@ -70,6 +101,11 @@ impl GlueCatalogProvider { self.config.endpoint_url.is_some() } + /// Returns the configured AWS Glue Data Catalog ID. + pub(super) fn catalog_id(&self) -> Option { + self.config.catalog_id.clone() + } + pub(super) fn database_name(database: &Namespace) -> CatalogResult { if database.tail.is_empty() { Ok(database.head.to_string()) @@ -163,21 +199,21 @@ impl GlueCatalogProvider { // Add partition columns for pk in table.partition_keys() { - if let Some(type_str) = pk.r#type() { - if let Ok(data_type) = glue_type_to_arrow(type_str) { - columns.push(TableColumnStatus { - name: pk.name().to_string(), - data_type, - nullable: true, - comment: pk.comment().map(|s| s.to_string()), - default: None, - generated_always_as: None, - identity: None, - is_partition: true, - is_bucket: false, - is_cluster: false, - }); - } + if let Some(type_str) = pk.r#type() + && let Ok(data_type) = glue_type_to_arrow(type_str) + { + columns.push(TableColumnStatus { + name: pk.name().to_string(), + data_type, + nullable: true, + comment: pk.comment().map(|s| s.to_string()), + default: None, + generated_always_as: None, + identity: None, + is_partition: true, + is_bucket: false, + is_cluster: false, + }); } } @@ -356,10 +392,7 @@ fn table_provider_format(parameters: Option<&HashMap>) -> Option .get(hive::SPARK_DATASOURCE_PROVIDER_KEY)? .trim() .to_ascii_lowercase(); - Some(match provider.as_str() { - "deltalake" => "delta".to_string(), - _ => provider, - }) + Some(provider) } #[async_trait::async_trait] @@ -407,6 +440,7 @@ impl CatalogProvider for GlueCatalogProvider { let result = client .create_database() + .set_catalog_id(self.catalog_id()) .database_input(database_input) .send() .await; @@ -437,7 +471,12 @@ impl CatalogProvider for GlueCatalogProvider { let client = self.get_client().await?; let database_name = Self::database_name(database)?; - let result = client.get_database().name(&database_name).send().await; + let result = client + .get_database() + .set_catalog_id(self.catalog_id()) + .name(&database_name) + .send() + .await; match result { Ok(output) => { @@ -469,7 +508,11 @@ impl CatalogProvider for GlueCatalogProvider { let client = self.get_client().await?; let mut databases = Vec::new(); - let mut paginator = client.get_databases().into_paginator().send(); + let mut paginator = client + .get_databases() + .set_catalog_id(self.catalog_id()) + .into_paginator() + .send(); while let Some(page) = paginator .next() @@ -506,7 +549,12 @@ impl CatalogProvider for GlueCatalogProvider { cascade: _, // Glue requires database to be empty; cascade not directly supported } = options; - let result = client.delete_database().name(&database_name).send().await; + let result = client + .delete_database() + .set_catalog_id(self.catalog_id()) + .name(&database_name) + .send() + .await; match result { Ok(_) => Ok(()), @@ -577,6 +625,7 @@ impl CatalogProvider for GlueCatalogProvider { let result = client .get_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .name(table) .send() @@ -621,6 +670,7 @@ impl CatalogProvider for GlueCatalogProvider { let mut tables = Vec::new(); let mut paginator = client .get_tables() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .into_paginator() .send(); @@ -662,6 +712,7 @@ impl CatalogProvider for GlueCatalogProvider { let result = client .delete_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .name(table) .send() @@ -697,6 +748,7 @@ impl CatalogProvider for GlueCatalogProvider { let database_name = Self::database_name(database)?; let result = client .get_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .name(table) .send() @@ -726,6 +778,7 @@ impl CatalogProvider for GlueCatalogProvider { let table_input = Self::table_input_with_parameters(&table_value, parameters)?; let mut update_table = client .update_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .table_input(table_input); if let Some(version_id) = table_value.version_id() { @@ -793,6 +846,7 @@ impl CatalogProvider for GlueCatalogProvider { let result = client .create_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .table_input(view_input) .send() @@ -826,6 +880,7 @@ impl CatalogProvider for GlueCatalogProvider { let result = client .get_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .name(view) .send() @@ -870,6 +925,7 @@ impl CatalogProvider for GlueCatalogProvider { let mut views = Vec::new(); let mut paginator = client .get_tables() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .into_paginator() .send(); @@ -903,6 +959,7 @@ impl CatalogProvider for GlueCatalogProvider { let result = client .delete_table() + .set_catalog_id(self.catalog_id()) .database_name(&database_name) .name(view) .send() @@ -961,7 +1018,7 @@ mod tests { .storage_descriptor(storage) .set_parameters(Some(HashMap::from([( hive::SPARK_DATASOURCE_PROVIDER_KEY.to_string(), - "deltalake".to_string(), + "delta".to_string(), )]))) .build() .unwrap(); diff --git a/crates/sail-catalog-glue/tests/common/mod.rs b/crates/sail-catalog-glue/tests/common/mod.rs index ef7a6fed4..b63e2b0a6 100644 --- a/crates/sail-catalog-glue/tests/common/mod.rs +++ b/crates/sail-catalog-glue/tests/common/mod.rs @@ -47,17 +47,19 @@ pub async fn setup_glue_catalog( let port = moto.get_host_port_ipv4(5000).await.expect("get port"); let endpoint = format!("http://{host}:{port}"); - // Set AWS credentials via environment for the test - // Moto accepts any credentials when endpoint_url is set - std::env::set_var("AWS_ACCESS_KEY_ID", "testing"); - std::env::set_var("AWS_SECRET_ACCESS_KEY", "testing"); - let config = GlueCatalogConfig { + catalog_id: Some("123456789012".to_string()), region: Some("us-east-1".to_string()), endpoint_url: Some(endpoint), }; - let provider = GlueCatalogProvider::new(test_name.to_string(), config); + let provider = GlueCatalogProvider::new_with_static_credentials( + test_name.to_string(), + config, + "testing".to_string(), + "testing".to_string(), + None, + ); (provider, moto) } diff --git a/crates/sail-catalog-glue/tests/database_tests.rs b/crates/sail-catalog-glue/tests/database_tests.rs index c7379f2b4..29cd9871d 100644 --- a/crates/sail-catalog-glue/tests/database_tests.rs +++ b/crates/sail-catalog-glue/tests/database_tests.rs @@ -53,10 +53,12 @@ async fn test_create_database() { ); assert_eq!(created_db.comment, Some("test comment".to_string())); assert_eq!(created_db.location, Some("s3://bucket/path".to_string())); - assert!(created_db - .properties - .iter() - .any(|(k, v)| k == "key1" && v == "value1")); + assert!( + created_db + .properties + .iter() + .any(|(k, v)| k == "key1" && v == "value1") + ); // Test creating duplicate without if_not_exists - should fail let result = catalog diff --git a/crates/sail-catalog-glue/tests/table_tests.rs b/crates/sail-catalog-glue/tests/table_tests.rs index 779f2325a..9d14472c0 100644 --- a/crates/sail-catalog-glue/tests/table_tests.rs +++ b/crates/sail-catalog-glue/tests/table_tests.rs @@ -105,9 +105,11 @@ async fn test_create_table() { transform: None, }] ); - assert!(properties - .iter() - .any(|(k, v)| k == "owner" && v == "test_user")); + assert!( + properties + .iter() + .any(|(k, v)| k == "owner" && v == "test_user") + ); } _ => panic!("Expected Table kind"), } diff --git a/crates/sail-catalog-glue/tests/view_tests.rs b/crates/sail-catalog-glue/tests/view_tests.rs index 549718f6b..0dd68ea77 100644 --- a/crates/sail-catalog-glue/tests/view_tests.rs +++ b/crates/sail-catalog-glue/tests/view_tests.rs @@ -102,9 +102,11 @@ async fn test_create_view() { assert_eq!(definition, "SELECT id, name, price FROM products"); assert_eq!(columns.len(), 3); assert_eq!(comment, &Some("View of products".to_string())); - assert!(properties - .iter() - .any(|(k, v)| k == "owner" && v == "test_user")); + assert!( + properties + .iter() + .any(|(k, v)| k == "owner" && v == "test_user") + ); } _ => panic!("Expected View kind"), } diff --git a/crates/sail-catalog-hms/src/convert.rs b/crates/sail-catalog-hms/src/convert.rs index 7c20630b2..0f1ee4aa4 100644 --- a/crates/sail-catalog-hms/src/convert.rs +++ b/crates/sail-catalog-hms/src/convert.rs @@ -11,10 +11,10 @@ use sail_catalog::provider::{ }; use sail_catalog::utils::quote_namespace_if_needed; use sail_common_datafusion::catalog::iceberg::{ - is_iceberg_table_properties, ICEBERG_TABLE_TYPE_KEY, ICEBERG_TABLE_TYPE_VALUE, + ICEBERG_TABLE_TYPE_KEY, ICEBERG_TABLE_TYPE_VALUE, is_iceberg_table_marker, }; use sail_common_datafusion::catalog::{ - identity_partition_fields, DatabaseStatus, TableColumnStatus, TableKind, TableStatus, + DatabaseStatus, TableColumnStatus, TableKind, TableStatus, identity_partition_fields, }; use sail_common_datafusion::column_features::{ ColumnFeatureKey, ColumnFeatures, ColumnFeaturesBuilder, @@ -32,6 +32,7 @@ pub(crate) const COMMENT_KEY: &str = "comment"; pub(crate) const EXTERNAL_KEY: &str = "EXTERNAL"; pub(crate) const EXTERNAL_TRUE: &str = "TRUE"; pub(crate) const SPARK_DATASOURCE_PROVIDER_KEY: &str = "spark.sql.sources.provider"; +pub(crate) const SPARK_DATASOURCE_PATH_KEY: &str = "path"; pub(crate) const SPARK_SCHEMA_KEY: &str = "spark.sql.sources.schema"; pub(crate) const EXTERNAL_TABLE_TYPE: &str = "EXTERNAL_TABLE"; pub(crate) const VIRTUAL_VIEW_TYPE: &str = "VIRTUAL_VIEW"; @@ -94,21 +95,11 @@ pub(crate) fn table_to_status( Some(spark_columns) => reorder_spark_columns(spark_columns, &partition_columns)?, None => columns_from_hms(storage, table.partition_keys.as_ref())?, }; - let location = storage - .and_then(|sd| sd.location.as_ref()) - .map(ToString::to_string); + let location = table_location(table); let format = if is_iceberg_table_parameters(table.parameters.as_ref()) { ICEBERG_TABLE_TYPE_VALUE.to_string() } else { - table_provider_format(table.parameters.as_ref()).unwrap_or_else(|| { - detect_hms_logical_format( - storage - .and_then(|sd| sd.serde_info.as_ref()) - .and_then(|serde| serde.serialization_lib.as_deref()), - storage.and_then(|sd| sd.input_format.as_deref()), - storage.and_then(|sd| sd.output_format.as_deref()), - ) - }) + resolve_table_format(table.parameters.as_ref(), storage) }; let partition_by = table .partition_keys @@ -210,10 +201,10 @@ pub(crate) fn build_generic_table( let mut parameters = vec_to_map(properties); insert_comment(&mut parameters, comment); let table_parameters = parameters.get_or_insert_with(AHashMap::new); - if format.logical_format.eq_ignore_ascii_case("delta") { + if let Some(provider) = spark_datasource_provider(format.logical_format) { table_parameters.insert( FastStr::from_static_str(SPARK_DATASOURCE_PROVIDER_KEY), - FastStr::from_static_str("delta"), + FastStr::from_string(provider), ); } else if format .logical_format @@ -229,14 +220,18 @@ pub(crate) fn build_generic_table( FastStr::from_static_str(EXTERNAL_KEY), FastStr::from_static_str(EXTERNAL_TRUE), ); - let serde_parameters = location.as_ref().map(|location| { - let mut parameters = AHashMap::new(); - parameters.insert( - FastStr::from_static_str("path"), - FastStr::from_string(location.clone()), - ); - parameters - }); + let serde_parameters = if is_spark_datasource_format(format.logical_format) { + location.as_ref().map(|location| { + let mut parameters = AHashMap::new(); + parameters.insert( + FastStr::from_static_str(SPARK_DATASOURCE_PATH_KEY), + FastStr::from_string(location.clone()), + ); + parameters + }) + } else { + None + }; Ok(Table { db_name: Some(database_name.to_string().into()), @@ -321,10 +316,12 @@ pub(crate) fn inject_spark_metadata( } } - parameters.insert( - FastStr::from_static_str(SPARK_DATASOURCE_PROVIDER_KEY), - FastStr::from_string(format.to_string()), - ); + if let Some(provider) = spark_datasource_provider(format) { + parameters.insert( + FastStr::from_static_str(SPARK_DATASOURCE_PROVIDER_KEY), + FastStr::from_string(provider), + ); + } parameters.insert( FastStr::from_static_str("spark.sql.create.version"), @@ -470,22 +467,59 @@ pub(crate) fn extract_property( parameters.and_then(|props| props.get(key).map(ToString::to_string)) } +fn is_spark_datasource_format(logical_format: &str) -> bool { + spark_datasource_provider(logical_format).is_some() +} + +// Generic formats use Spark's datasource representation unless the catalog +// contract explicitly selects Hive SerDe or Iceberg metadata. This decision is +// independent of Sail's current execution support for the provider. +fn spark_datasource_provider(logical_format: &str) -> Option { + let provider = logical_format.trim().to_ascii_lowercase(); + match provider.as_str() { + "textfile" | ICEBERG_TABLE_TYPE_VALUE => None, + _ => Some(provider), + } +} + +pub(crate) fn table_location(table: &Table) -> Option { + let storage = table.sd.as_ref()?; + if extract_property(table.parameters.as_ref(), SPARK_DATASOURCE_PROVIDER_KEY).is_some() { + return storage + .serde_info + .as_ref() + .and_then(|serde| { + get_case_insensitive(serde.parameters.as_ref(), SPARK_DATASOURCE_PATH_KEY) + }) + .map(ToString::to_string); + } + storage.location.as_ref().map(ToString::to_string) +} + fn table_provider_format(parameters: Option<&AHashMap>) -> Option { let provider = extract_property(parameters, SPARK_DATASOURCE_PROVIDER_KEY)?; let provider = provider.trim().to_ascii_lowercase(); - Some(match provider.as_str() { - "deltalake" => "delta".to_string(), - _ => provider, - }) + match provider.as_str() { + "" => Some("unknown".to_string()), + _ => Some(provider), + } +} + +fn get_case_insensitive<'a>( + parameters: Option<&'a AHashMap>, + key: &str, +) -> Option<&'a str> { + parameters? + .iter() + .find(|(candidate, _)| candidate.as_str().eq_ignore_ascii_case(key)) + .map(|(_, value)| value.as_str()) } fn is_iceberg_table_parameters(parameters: Option<&AHashMap>) -> bool { parameters.is_some_and(|properties| { - is_iceberg_table_properties( - properties - .iter() - .map(|(key, value)| (key.as_str(), value.as_str())), - ) + properties + .iter() + .any(|(key, value)| is_iceberg_table_marker(key.as_str().trim(), value.as_str().trim())) }) } @@ -665,15 +699,33 @@ fn reorder_spark_columns( Ok(regular.into_iter().chain(partition).collect()) } -fn detect_hms_logical_format( - serde_library: Option<&str>, - input_format: Option<&str>, - output_format: Option<&str>, +/// Resolve a non-Iceberg table's logical format from its HMS metadata. +/// +/// The domain model, in priority order: +/// +/// 1. A Spark datasource provider is preserved after trimming and normalizing +/// case. +/// 2. Only tables without a datasource marker are classified from their Hive +/// SerDe/InputFormat/OutputFormat metadata. +/// 3. If neither is present or detectable, the format is `unknown`. +fn resolve_table_format( + parameters: Option<&AHashMap>, + storage: Option<&StorageDescriptor>, ) -> String { - match HiveDetectedFormat::detect(serde_library, input_format, output_format) { - HiveDetectedFormat::Csv => "csv".to_string(), - detected => detected.as_str().to_string(), + if let Some(format) = table_provider_format(parameters) { + return format; } + let detected = HiveDetectedFormat::detect( + storage + .and_then(|sd| sd.serde_info.as_ref()) + .and_then(|serde| serde.serialization_lib.as_deref()), + storage.and_then(|sd| sd.input_format.as_deref()), + storage.and_then(|sd| sd.output_format.as_deref()), + ); + if detected != HiveDetectedFormat::Unknown { + return detected.as_str().to_string(); + } + "unknown".to_string() } fn insert_comment(parameters: &mut Option>, comment: Option) { @@ -836,9 +888,10 @@ mod tests { use sail_common_hms::hms::{FieldSchema, SerDeInfo, StorageDescriptor, Table}; use super::{ - build_generic_table, build_view, columns_from_spark_properties, database_to_status, - inject_spark_metadata, is_view_table, map_to_vec, validate_namespace, GenericTableFormat, - COMMENT_KEY, SPARK_DATASOURCE_PROVIDER_KEY, SPARK_SCHEMA_KEY, VIRTUAL_VIEW_TYPE, + COMMENT_KEY, GenericTableFormat, SPARK_DATASOURCE_PATH_KEY, SPARK_DATASOURCE_PROVIDER_KEY, + SPARK_SCHEMA_KEY, VIRTUAL_VIEW_TYPE, build_generic_table, build_view, + columns_from_spark_properties, database_to_status, inject_spark_metadata, is_view_table, + map_to_vec, validate_namespace, }; #[test] @@ -893,16 +946,78 @@ mod tests { Some(super::EXTERNAL_TABLE_TYPE) ); let properties = map_to_vec(table.parameters.as_ref()); - assert!(properties - .iter() - .any(|(k, v)| k == super::EXTERNAL_KEY && v == super::EXTERNAL_TRUE)); - assert!(properties - .iter() - .any(|(k, v)| k == COMMENT_KEY && v == "comment")); + assert!( + properties + .iter() + .any(|(k, v)| k == super::EXTERNAL_KEY && v == super::EXTERNAL_TRUE) + ); + assert!( + properties + .iter() + .any(|(k, v)| k == COMMENT_KEY && v == "comment") + ); + } + + #[test] + fn test_build_generic_table_writes_datasource_location_as_serde_path() { + for (logical_format, storage) in [ + ("delta", HiveStorageFormat::parquet()), + ("parquet", HiveStorageFormat::parquet()), + ("csv", HiveStorageFormat::csv()), + ("json", HiveStorageFormat::json()), + ("avro", HiveStorageFormat::avro()), + ("orc", HiveStorageFormat::orc()), + ] { + let table = build_generic_table( + "default", + "items", + vec![CreateTableColumnOptions { + name: "id".to_string(), + data_type: DataType::Int64, + nullable: false, + comment: None, + default: None, + generated_always_as: None, + identity: None, + }], + vec![], + Some(format!("s3://warehouse/{logical_format}_items")), + GenericTableFormat { + logical_format, + storage: &storage, + }, + None, + vec![], + ) + .unwrap(); + + let table_parameters = map_to_vec(table.parameters.as_ref()); + assert!(table_parameters.iter().any(|(key, value)| { + key == SPARK_DATASOURCE_PROVIDER_KEY && value == logical_format + })); + + let serde_parameters = table + .sd + .as_ref() + .and_then(|sd| sd.serde_info.as_ref()) + .and_then(|serde| serde.parameters.as_ref()) + .expect("datasource serde parameters"); + assert_eq!( + serde_parameters + .get(SPARK_DATASOURCE_PATH_KEY) + .map(|value| value.as_str()), + Some(format!("s3://warehouse/{logical_format}_items").as_str()) + ); + assert!( + !table_parameters + .iter() + .any(|(key, _)| key == SPARK_DATASOURCE_PATH_KEY) + ); + } } #[test] - fn test_build_generic_table_marks_delta_provider() { + fn test_build_generic_table_textfile_does_not_write_datasource_serde_path() { let table = build_generic_table( "default", "items", @@ -918,18 +1033,46 @@ mod tests { vec![], Some("s3://warehouse/items".to_string()), GenericTableFormat { - logical_format: "delta", - storage: &HiveStorageFormat::parquet(), + logical_format: "textfile", + storage: &HiveStorageFormat::textfile(), }, None, vec![], ) .unwrap(); - let properties = map_to_vec(table.parameters.as_ref()); - assert!(properties - .iter() - .any(|(k, v)| k == SPARK_DATASOURCE_PROVIDER_KEY && v == "delta")); + // Hive's native textfile format is read back via SerDe detection, so + // build_generic_table must not register a Spark datasource provider. + let table_parameters = map_to_vec(table.parameters.as_ref()); + assert!( + !table_parameters + .iter() + .any(|(key, _)| key == SPARK_DATASOURCE_PROVIDER_KEY), + "build_generic_table must not insert a datasource provider for textfile" + ); + + // The location must also not be mirrored into the SerDe parameters as a + // datasource `path`. + let serde_path = table + .sd + .as_ref() + .and_then(|sd| sd.serde_info.as_ref()) + .and_then(|serde| serde.parameters.as_ref()) + .and_then(|params| { + params + .get(SPARK_DATASOURCE_PATH_KEY) + .map(ToString::to_string) + }); + assert!( + serde_path.is_none(), + "textfile must not write a datasource serde path, got {serde_path:?}" + ); + + // The requested location is still stored on the storage descriptor. + assert_eq!( + table.sd.as_ref().and_then(|sd| sd.location.as_deref()), + Some("s3://warehouse/items") + ); } #[test] @@ -1152,15 +1295,73 @@ mod tests { let props = map_to_vec(table.parameters.as_ref()); assert!(props.iter().any(|(k, v)| k == "owner" && v == "alice")); assert!(props.iter().any(|(k, _)| k == SPARK_SCHEMA_KEY)); - assert!(props - .iter() - .any(|(k, v)| k == "spark.sql.sources.schema.partCol.0" && v == "day")); - assert!(props - .iter() - .any(|(k, v)| k == "spark.sql.sources.provider" && v == "parquet")); + assert!( + props + .iter() + .any(|(k, v)| k == "spark.sql.sources.schema.partCol.0" && v == "day") + ); + assert!( + props + .iter() + .any(|(k, v)| k == "spark.sql.sources.provider" && v == "parquet") + ); assert!(props.iter().any(|(k, _)| k == "spark.sql.create.version")); } + #[test] + fn test_inject_spark_metadata_skips_datasource_provider_for_textfile() { + let mut table = build_generic_table( + "default", + "items", + vec![CreateTableColumnOptions { + name: "id".to_string(), + data_type: DataType::Int64, + nullable: false, + comment: None, + default: None, + generated_always_as: None, + identity: None, + }], + vec![], + Some("s3://warehouse/items".to_string()), + GenericTableFormat { + logical_format: "textfile", + storage: &HiveStorageFormat::textfile(), + }, + None, + vec![], + ) + .unwrap(); + + inject_spark_metadata( + &mut table, + &[CreateTableColumnOptions { + name: "id".to_string(), + data_type: DataType::Int64, + nullable: false, + comment: None, + default: None, + generated_always_as: None, + identity: None, + }], + &[], + "textfile", + ) + .unwrap(); + + let props = map_to_vec(table.parameters.as_ref()); + assert!( + !props + .iter() + .any(|(k, _)| k == SPARK_DATASOURCE_PROVIDER_KEY), + "textfile must not get a datasource provider key, got: {props:?}" + ); + assert!( + props.iter().any(|(k, _)| k == SPARK_SCHEMA_KEY), + "Spark schema must still be written for textfile" + ); + } + #[test] fn test_spark_metadata_round_trips_column_features() { let columns = vec![ @@ -1477,7 +1678,7 @@ mod tests { ); parameters.insert( FastStr::from_static_str("spark.sql.sources.provider"), - FastStr::from_static_str("deltalake"), + FastStr::from_static_str("delta"), ); parameters.insert( FastStr::from_static_str("spark.sql.sources.schema.numPartCols"), @@ -1576,9 +1777,11 @@ mod tests { let error = super::table_to_status("hms", &namespace, &table).unwrap_err(); - assert!(error - .to_string() - .contains("Partition columns missing in Spark schema metadata: day")); + assert!( + error + .to_string() + .contains("Partition columns missing in Spark schema metadata: day") + ); } #[test] @@ -1667,9 +1870,11 @@ mod tests { let error = super::table_to_status("hms", &namespace, &table).unwrap_err(); - assert!(error - .to_string() - .contains("Missing split property part spark.sql.sources.schema.part.0")); + assert!( + error + .to_string() + .contains("Missing split property part spark.sql.sources.schema.part.0") + ); } #[test] @@ -2114,6 +2319,191 @@ mod tests { } } + fn table_status_location(table: &Table) -> Option { + let namespace = sail_catalog::provider::Namespace::try_from(vec!["default"]).unwrap(); + let status = super::table_to_status("hms", &namespace, table).unwrap(); + match status.kind { + sail_common_datafusion::catalog::TableKind::Table { location, .. } => location, + other => panic!("expected table, got {other:?}"), + } + } + + fn table_status_format(table: &Table) -> Option { + let namespace = sail_catalog::provider::Namespace::try_from(vec!["default"]).unwrap(); + let status = super::table_to_status("hms", &namespace, table).unwrap(); + match status.kind { + sail_common_datafusion::catalog::TableKind::Table { format, .. } => Some(format), + other => panic!("expected table, got {other:?}"), + } + } + + fn hms_table_with_locations( + provider: Option<&str>, + sd_location: Option<&str>, + serde_path_key: Option<&str>, + serde_path: Option<&str>, + ) -> Table { + let mut table_parameters = AHashMap::new(); + if let Some(provider) = provider { + table_parameters.insert( + FastStr::from_static_str(SPARK_DATASOURCE_PROVIDER_KEY), + FastStr::from_string(provider.to_string()), + ); + } + + let serde_parameters = serde_path_key.map(|key| { + [( + FastStr::from_string(key.to_string()), + FastStr::from_string(serde_path.unwrap_or_default().to_string()), + )] + .into_iter() + .collect() + }); + + Table { + db_name: Some("default".into()), + table_name: Some("items".into()), + parameters: if table_parameters.is_empty() { + None + } else { + Some(table_parameters) + }, + sd: Some(StorageDescriptor { + cols: Some(vec![FieldSchema { + name: Some("id".into()), + r#type: Some("bigint".into()), + ..Default::default() + }]), + location: sd_location.map(|location| location.to_string().into()), + input_format: Some(HiveStorageFormat::parquet().input_format.into()), + output_format: Some(HiveStorageFormat::parquet().output_format.into()), + serde_info: Some(SerDeInfo { + serialization_lib: Some(HiveStorageFormat::parquet().serde_library.into()), + parameters: serde_parameters, + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } + + #[test] + fn test_table_to_status_uses_datasource_serde_path_for_any_provider() { + for provider in ["delta", "parquet", "csv", "json", "avro"] { + let table = hms_table_with_locations( + Some(provider), + Some("s3://wrong-or-sentinel"), + Some("path"), + Some(&format!("s3://actual-{provider}")), + ); + + assert_eq!( + table_status_location(&table).as_deref(), + Some(format!("s3://actual-{provider}").as_str()) + ); + assert_eq!(table_status_format(&table).as_deref(), Some(provider)); + } + } + + #[test] + fn test_table_to_status_preserves_unsupported_datasource_providers_over_serde() { + for (provider, expected) in [ + ("com.acme.CustomSource", "com.acme.customsource"), + ("orc", "orc"), + ] { + let table = hms_table_with_locations( + Some(provider), + Some("s3://sd-location"), + Some("path"), + Some(&format!("s3://actual-{provider}")), + ); + let namespace = sail_catalog::provider::Namespace::try_from(vec!["default"]).unwrap(); + + let status = super::table_to_status("hms", &namespace, &table).unwrap(); + match status.kind { + sail_common_datafusion::catalog::TableKind::Table { format, .. } => { + assert_eq!( + format, expected, + "datasource provider {provider} must remain authoritative over Parquet SerDe metadata" + ); + } + other => panic!("expected table, got {other:?}"), + } + } + } + + #[test] + fn test_table_to_status_datasource_provider_without_path_has_no_location() { + let missing = hms_table_with_locations(Some("delta"), Some("s3://sd-location"), None, None); + assert_eq!( + table_status_location(&missing).as_deref(), + None, + "datasource tables without a SerDe path must not fall back to sd.location" + ); + } + + #[test] + fn test_table_to_status_preserves_empty_datasource_path() { + let empty = hms_table_with_locations( + Some("delta"), + Some("s3://sd-location"), + Some("path"), + Some(""), + ); + assert_eq!( + table_status_location(&empty).as_deref(), + Some(""), + "a present empty datasource path must remain authoritative" + ); + } + + #[test] + fn test_table_to_status_ignores_serde_path_for_hive_serde_table() { + let table = hms_table_with_locations( + None, + Some("s3://hive-sd-location"), + Some("path"), + Some("s3://datasource-path"), + ); + + assert_eq!( + table_status_location(&table).as_deref(), + Some("s3://hive-sd-location") + ); + } + + #[test] + fn test_table_to_status_reads_datasource_path_key_case_insensitively() { + let table = hms_table_with_locations( + Some(" DeLtA "), + Some("s3://wrong-or-sentinel"), + Some("PATH"), + Some("s3://actual-uppercase"), + ); + + assert_eq!( + table_status_location(&table).as_deref(), + Some("s3://actual-uppercase") + ); + assert_eq!(table_status_format(&table).as_deref(), Some("delta")); + } + + #[test] + fn test_table_to_status_metadata_location_without_marker_is_not_iceberg() { + let mut table = hms_table_with_locations(None, Some("s3://warehouse/items"), None, None); + table.parameters = Some(AHashMap::from_iter([( + FastStr::from_static_str("metadata_location"), + FastStr::from_static_str("s3://warehouse/items/metadata/00001.json"), + )])); + + assert_eq!( + table_status_format(&table).as_deref(), + Some("parquet"), + "metadata_location alone must not classify an HMS table as Iceberg" + ); + } + #[test] fn test_build_view_marks_virtual_view() { let table = build_view( diff --git a/crates/sail-catalog-hms/src/data_type.rs b/crates/sail-catalog-hms/src/data_type.rs index fbd973509..025854007 100644 --- a/crates/sail-catalog-hms/src/data_type.rs +++ b/crates/sail-catalog-hms/src/data_type.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; use sail_catalog::error::{CatalogError, CatalogResult}; use sail_common_datafusion::column_features::ColumnFeatureKey; -use serde_json::{json, Value}; +use serde_json::{Value, json}; pub fn arrow_to_hive_type(data_type: &DataType) -> CatalogResult { match data_type { @@ -74,14 +74,14 @@ pub fn arrow_to_hive_type(data_type: &DataType) -> CatalogResult { Ok(format!("struct<{}>", fields?.join(","))) } DataType::Map(field, _) => { - if let DataType::Struct(fields) = field.data_type() { - if fields.len() == 2 { - return Ok(format!( - "map<{},{}>", - arrow_to_hive_type(fields[0].data_type())?, - arrow_to_hive_type(fields[1].data_type())? - )); - } + if let DataType::Struct(fields) = field.data_type() + && fields.len() == 2 + { + return Ok(format!( + "map<{},{}>", + arrow_to_hive_type(fields[0].data_type())?, + arrow_to_hive_type(fields[1].data_type())? + )); } Err(CatalogError::InvalidArgument( "Map type must have key and value fields".to_string(), @@ -746,9 +746,11 @@ mod tests { fn test_spark_struct_json_rejects_malformed_json() { let error = super::spark_struct_json_to_fields("{\"type\":\"struct\",\"fields\":[").unwrap_err(); - assert!(error - .to_string() - .contains("Failed to parse Spark schema JSON")); + assert!( + error + .to_string() + .contains("Failed to parse Spark schema JSON") + ); } #[test] diff --git a/crates/sail-catalog-hms/src/managed_table.rs b/crates/sail-catalog-hms/src/managed_table.rs index 0bc263ea7..4e25bd0ae 100644 --- a/crates/sail-catalog-hms/src/managed_table.rs +++ b/crates/sail-catalog-hms/src/managed_table.rs @@ -12,7 +12,7 @@ use sail_common_hms::hms::{ }; use volo_thrift::MaybeException; -use crate::provider::{apply_alter_table_options, HmsCatalogProvider}; +use crate::provider::{HmsCatalogProvider, apply_alter_table_options}; const HMS_LOCK_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(30); const HMS_LOCK_CHECK_INTERVAL: Duration = Duration::from_millis(200); diff --git a/crates/sail-catalog-hms/src/provider.rs b/crates/sail-catalog-hms/src/provider.rs index 6d68311d5..0b62c9b9f 100644 --- a/crates/sail-catalog-hms/src/provider.rs +++ b/crates/sail-catalog-hms/src/provider.rs @@ -5,10 +5,10 @@ use pilota::{AHashMap, FastStr}; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::hive_format::HiveCatalogFormat; use sail_catalog::provider::{ - plain_lakehouse_create_table_metadata_requirement, AlterTableOptions, CatalogProvider, - CreateDatabaseOptions, CreateTableMetadataRequirement, CreateTableOptions, CreateViewOptions, - DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, PartitionTransform, - TableFormatCreateMetadataMode, + AlterTableOptions, CatalogProvider, CreateDatabaseOptions, CreateTableMetadataRequirement, + CreateTableOptions, CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, + Namespace, PartitionTransform, TableFormatCreateMetadataMode, + plain_lakehouse_create_table_metadata_requirement, }; use sail_common::runtime::RuntimeHandle; use sail_common_datafusion::catalog::{DatabaseStatus, TableStatus}; @@ -25,10 +25,9 @@ use tokio::sync::Mutex; use volo_thrift::MaybeException; use crate::convert::{ - alter_spark_column_default, build_database, build_generic_table, build_view, - database_to_status, inject_spark_metadata, is_view_table, reject_spark_properties, + GenericTableFormat, alter_spark_column_default, build_database, build_generic_table, + build_view, database_to_status, inject_spark_metadata, is_view_table, reject_spark_properties, reject_spark_property_keys, table_to_status, validate_namespace, view_to_status, - GenericTableFormat, }; use crate::data_type::arrow_to_hive_type; use crate::managed_table; @@ -420,7 +419,7 @@ impl HmsCatalogProvider { (_, other) => { return Err(CatalogError::InvalidArgument(format!( "Unsupported thrift_transport '{other}', expected 'buffered' or 'framed'" - ))) + ))); } }; Ok(client) @@ -461,10 +460,10 @@ impl HmsCatalogProvider { ) -> CatalogResult<(usize, ThriftHiveMetastoreClient)> { { let state = self.state.lock().await; - if state.active_index != failed_index { - if let Some(client) = &state.client { - return Ok((state.active_index, client.clone())); - } + if state.active_index != failed_index + && let Some(client) = &state.client + { + return Ok((state.active_index, client.clone())); } } @@ -1315,11 +1314,13 @@ mod tests { assert!(matches!(error, CatalogError::InvalidArgument(_))); assert!(table.parameters.as_ref().unwrap().contains_key("owner")); - assert!(!table - .parameters - .as_ref() - .unwrap() - .contains_key("spark.sql.sources.provider")); + assert!( + !table + .parameters + .as_ref() + .unwrap() + .contains_key("spark.sql.sources.provider") + ); } #[test] @@ -1347,11 +1348,13 @@ mod tests { .unwrap_err(); assert!(matches!(error, CatalogError::InvalidArgument(_))); - assert!(table - .parameters - .as_ref() - .unwrap() - .contains_key("spark.sql.sources.provider")); + assert!( + table + .parameters + .as_ref() + .unwrap() + .contains_key("spark.sql.sources.provider") + ); } #[test] @@ -1605,9 +1608,11 @@ mod tests { let error = super::split_hms_uri_list(&[]).unwrap_err(); assert!(matches!(error, CatalogError::InvalidArgument(_))); - assert!(error - .to_string() - .contains("must contain at least one endpoint")); + assert!( + error + .to_string() + .contains("must contain at least one endpoint") + ); } #[test] diff --git a/crates/sail-catalog-hms/src/security/gssapi.rs b/crates/sail-catalog-hms/src/security/gssapi.rs index bb55642fb..b081f0b81 100644 --- a/crates/sail-catalog-hms/src/security/gssapi.rs +++ b/crates/sail-catalog-hms/src/security/gssapi.rs @@ -623,7 +623,7 @@ mod tests { use sail_catalog::error::CatalogError; use super::{ - load_gssapi_library_by_name_for_test, select_security_layer, SaslQop, SecurityLayer, + SaslQop, SecurityLayer, load_gssapi_library_by_name_for_test, select_security_layer, }; fn build_error_message(name: &str) -> String { diff --git a/crates/sail-catalog-hms/src/security/gssapi_bindings.rs b/crates/sail-catalog-hms/src/security/gssapi_bindings.rs index 7fa246e5b..e969c5a9a 100644 --- a/crates/sail-catalog-hms/src/security/gssapi_bindings.rs +++ b/crates/sail-catalog-hms/src/security/gssapi_bindings.rs @@ -755,14 +755,14 @@ impl GSSAPI { pub unsafe fn new

(path: P) -> Result where P: AsRef<::std::ffi::OsStr>, - { + { unsafe { let library = ::libloading::Library::new(path)?; Self::from_library(library) - } + }} pub unsafe fn from_library(library: L) -> Result where L: Into<::libloading::Library>, - { + { unsafe { let __library = library.into(); let GSS_C_NT_USER_NAME = __library .get::<*mut gss_OID>(b"GSS_C_NT_USER_NAME\0") @@ -937,7 +937,7 @@ impl GSSAPI { gss_krb5_free_lucid_sec_context, gss_krb5_ui, }) - } + }} pub unsafe fn GSS_C_NT_USER_NAME(&self) -> *mut gss_OID { *self .GSS_C_NT_USER_NAME @@ -990,24 +990,24 @@ impl GSSAPI { arg6: *mut gss_cred_id_t, arg7: *mut gss_OID_set, arg8: *mut OM_uint32, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_acquire_cred .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, ) - } + }} pub unsafe fn gss_release_cred( &self, arg1: *mut OM_uint32, arg2: *mut gss_cred_id_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_release_cred .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} pub unsafe fn gss_init_sec_context( &self, arg1: *mut OM_uint32, @@ -1023,14 +1023,14 @@ impl GSSAPI { arg11: gss_buffer_t, arg12: *mut OM_uint32, arg13: *mut OM_uint32, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_init_sec_context .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, ) - } + }} pub unsafe fn gss_accept_sec_context( &self, arg1: *mut OM_uint32, @@ -1044,47 +1044,47 @@ impl GSSAPI { arg9: *mut OM_uint32, arg10: *mut OM_uint32, arg11: *mut gss_cred_id_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_accept_sec_context .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, ) - } + }} pub unsafe fn gss_process_context_token( &self, arg1: *mut OM_uint32, arg2: gss_ctx_id_t, arg3: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_process_context_token .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_delete_sec_context( &self, arg1: *mut OM_uint32, arg2: *mut gss_ctx_id_t, arg3: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_delete_sec_context .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_context_time( &self, arg1: *mut OM_uint32, arg2: gss_ctx_id_t, arg3: *mut OM_uint32, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_context_time .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_get_mic( &self, arg1: *mut OM_uint32, @@ -1092,12 +1092,12 @@ impl GSSAPI { arg3: gss_qop_t, arg4: gss_buffer_t, arg5: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_get_mic .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5) - } + }} pub unsafe fn gss_verify_mic( &self, arg1: *mut OM_uint32, @@ -1105,12 +1105,12 @@ impl GSSAPI { arg3: gss_buffer_t, arg4: gss_buffer_t, arg5: *mut gss_qop_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_verify_mic .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5) - } + }} pub unsafe fn gss_wrap( &self, arg1: *mut OM_uint32, @@ -1120,14 +1120,14 @@ impl GSSAPI { arg5: gss_buffer_t, arg6: *mut ::std::os::raw::c_int, arg7: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_wrap .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7 ) - } + }} pub unsafe fn gss_unwrap( &self, arg1: *mut OM_uint32, @@ -1136,12 +1136,12 @@ impl GSSAPI { arg4: gss_buffer_t, arg5: *mut ::std::os::raw::c_int, arg6: *mut gss_qop_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_unwrap .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5, arg6) - } + }} pub unsafe fn gss_display_status( &self, arg1: *mut OM_uint32, @@ -1150,84 +1150,84 @@ impl GSSAPI { arg4: gss_OID, arg5: *mut OM_uint32, arg6: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_display_status .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5, arg6) - } + }} pub unsafe fn gss_indicate_mechs( &self, arg1: *mut OM_uint32, arg2: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_indicate_mechs .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} pub unsafe fn gss_compare_name( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: gss_name_t, arg4: *mut ::std::os::raw::c_int, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_compare_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4) - } + }} pub unsafe fn gss_display_name( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: gss_buffer_t, arg4: *mut gss_OID, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_display_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4) - } + }} pub unsafe fn gss_import_name( &self, arg1: *mut OM_uint32, arg2: gss_buffer_t, arg3: gss_OID, arg4: *mut gss_name_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_import_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4) - } + }} pub unsafe fn gss_release_name( &self, arg1: *mut OM_uint32, arg2: *mut gss_name_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_release_name .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } - pub unsafe fn gss_release_buffer(&self, arg1: *mut OM_uint32, arg2: gss_buffer_t) -> OM_uint32 { + }} + pub unsafe fn gss_release_buffer(&self, arg1: *mut OM_uint32, arg2: gss_buffer_t) -> OM_uint32 { unsafe { (self .gss_release_buffer .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} pub unsafe fn gss_release_oid_set( &self, arg1: *mut OM_uint32, arg2: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_release_oid_set .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} pub unsafe fn gss_inquire_cred( &self, arg1: *mut OM_uint32, @@ -1236,12 +1236,12 @@ impl GSSAPI { arg4: *mut OM_uint32, arg5: *mut gss_cred_usage_t, arg6: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_inquire_cred .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5, arg6) - } + }} pub unsafe fn gss_inquire_context( &self, arg1: *mut OM_uint32, @@ -1253,14 +1253,14 @@ impl GSSAPI { arg7: *mut OM_uint32, arg8: *mut ::std::os::raw::c_int, arg9: *mut ::std::os::raw::c_int, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_inquire_context .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, ) - } + }} pub unsafe fn gss_wrap_size_limit( &self, arg1: *mut OM_uint32, @@ -1269,24 +1269,24 @@ impl GSSAPI { arg4: gss_qop_t, arg5: OM_uint32, arg6: *mut OM_uint32, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_wrap_size_limit .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5, arg6) - } + }} pub unsafe fn gss_export_name_object( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: gss_OID, arg4: *mut *mut ::std::os::raw::c_void, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_export_name_object .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4) - } + }} pub unsafe fn gss_add_cred( &self, arg1: *mut OM_uint32, @@ -1300,14 +1300,14 @@ impl GSSAPI { arg9: *mut gss_OID_set, arg10: *mut OM_uint32, arg11: *mut OM_uint32, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_add_cred .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, ) - } + }} pub unsafe fn gss_inquire_cred_by_mech( &self, arg1: *mut OM_uint32, @@ -1317,119 +1317,119 @@ impl GSSAPI { arg5: *mut OM_uint32, arg6: *mut OM_uint32, arg7: *mut gss_cred_usage_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_inquire_cred_by_mech .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7 ) - } + }} pub unsafe fn gss_export_sec_context( &self, arg1: *mut OM_uint32, arg2: *mut gss_ctx_id_t, arg3: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_export_sec_context .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_import_sec_context( &self, arg1: *mut OM_uint32, arg2: gss_buffer_t, arg3: *mut gss_ctx_id_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_import_sec_context .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } - pub unsafe fn gss_release_oid(&self, arg1: *mut OM_uint32, arg2: *mut gss_OID) -> OM_uint32 { + }} + pub unsafe fn gss_release_oid(&self, arg1: *mut OM_uint32, arg2: *mut gss_OID) -> OM_uint32 { unsafe { (self .gss_release_oid .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} pub unsafe fn gss_create_empty_oid_set( &self, arg1: *mut OM_uint32, arg2: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_create_empty_oid_set .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} pub unsafe fn gss_add_oid_set_member( &self, arg1: *mut OM_uint32, arg2: gss_OID, arg3: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_add_oid_set_member .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_test_oid_set_member( &self, arg1: *mut OM_uint32, arg2: gss_OID, arg3: gss_OID_set, arg4: *mut ::std::os::raw::c_int, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_test_oid_set_member .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4) - } + }} pub unsafe fn gss_str_to_oid( &self, arg1: *mut OM_uint32, arg2: gss_buffer_t, arg3: *mut gss_OID, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_str_to_oid .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_oid_to_str( &self, arg1: *mut OM_uint32, arg2: gss_OID, arg3: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_oid_to_str .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_inquire_names_for_mech( &self, arg1: *mut OM_uint32, arg2: gss_OID, arg3: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_inquire_names_for_mech .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_inquire_mechs_for_name( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: *mut gss_OID_set, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_inquire_mechs_for_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_sign( &self, arg1: *mut OM_uint32, @@ -1437,12 +1437,12 @@ impl GSSAPI { arg3: ::std::os::raw::c_int, arg4: gss_buffer_t, arg5: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_sign .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5) - } + }} pub unsafe fn gss_verify( &self, arg1: *mut OM_uint32, @@ -1450,12 +1450,12 @@ impl GSSAPI { arg3: gss_buffer_t, arg4: gss_buffer_t, arg5: *mut ::std::os::raw::c_int, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_verify .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5) - } + }} pub unsafe fn gss_seal( &self, arg1: *mut OM_uint32, @@ -1465,14 +1465,14 @@ impl GSSAPI { arg5: gss_buffer_t, arg6: *mut ::std::os::raw::c_int, arg7: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_seal .as_ref() .expect("Expected function, got error."))( arg1, arg2, arg3, arg4, arg5, arg6, arg7 ) - } + }} pub unsafe fn gss_unseal( &self, arg1: *mut OM_uint32, @@ -1481,46 +1481,46 @@ impl GSSAPI { arg4: gss_buffer_t, arg5: *mut ::std::os::raw::c_int, arg6: *mut ::std::os::raw::c_int, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_unseal .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4, arg5, arg6) - } + }} pub unsafe fn gss_export_name( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: gss_buffer_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_export_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_duplicate_name( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: *mut gss_name_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_duplicate_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3) - } + }} pub unsafe fn gss_canonicalize_name( &self, arg1: *mut OM_uint32, arg2: gss_name_t, arg3: gss_OID, arg4: *mut gss_name_t, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_canonicalize_name .as_ref() .expect("Expected function, got error."))(arg1, arg2, arg3, arg4) - } + }} pub unsafe fn GSS_KRB5_NT_PRINCIPAL_NAME(&self) -> *mut *const gss_OID_desc { *self .GSS_KRB5_NT_PRINCIPAL_NAME @@ -1574,76 +1574,76 @@ impl GSSAPI { minor_status: *mut OM_uint32, context_handle: gss_ctx_id_t, ticket_flags: *mut krb5_flags, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_krb5_get_tkt_flags .as_ref() .expect("Expected function, got error."))( minor_status, context_handle, ticket_flags ) - } + }} pub unsafe fn gss_krb5_copy_ccache( &self, minor_status: *mut OM_uint32, cred_handle: gss_cred_id_t, out_ccache: krb5_ccache, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_krb5_copy_ccache .as_ref() .expect("Expected function, got error."))(minor_status, cred_handle, out_ccache) - } + }} pub unsafe fn gss_krb5_ccache_name( &self, minor_status: *mut OM_uint32, name: *const ::std::os::raw::c_char, out_name: *mut *const ::std::os::raw::c_char, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_krb5_ccache_name .as_ref() .expect("Expected function, got error."))(minor_status, name, out_name) - } + }} pub unsafe fn gss_krb5_set_allowable_enctypes( &self, minor_status: *mut OM_uint32, cred: gss_cred_id_t, num_ktypes: OM_uint32, ktypes: *mut krb5_enctype, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_krb5_set_allowable_enctypes .as_ref() .expect("Expected function, got error."))(minor_status, cred, num_ktypes, ktypes) - } + }} pub unsafe fn gss_krb5_export_lucid_sec_context( &self, minor_status: *mut OM_uint32, context_handle: *mut gss_ctx_id_t, version: OM_uint32, kctx: *mut *mut ::std::os::raw::c_void, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_krb5_export_lucid_sec_context .as_ref() .expect("Expected function, got error."))( minor_status, context_handle, version, kctx ) - } + }} pub unsafe fn gss_krb5_free_lucid_sec_context( &self, minor_status: *mut OM_uint32, kctx: *mut ::std::os::raw::c_void, - ) -> OM_uint32 { + ) -> OM_uint32 { unsafe { (self .gss_krb5_free_lucid_sec_context .as_ref() .expect("Expected function, got error."))(minor_status, kctx) - } - pub unsafe fn gss_krb5_ui(&self, arg1: *mut OM_uint32, arg2: OM_uint32) -> OM_uint32 { + }} + pub unsafe fn gss_krb5_ui(&self, arg1: *mut OM_uint32, arg2: OM_uint32) -> OM_uint32 { unsafe { (self .gss_krb5_ui .as_ref() .expect("Expected function, got error."))(arg1, arg2) - } + }} } diff --git a/crates/sail-catalog-hms/src/security/thrift_sasl.rs b/crates/sail-catalog-hms/src/security/thrift_sasl.rs index 0a08aa610..451dd6dae 100644 --- a/crates/sail-catalog-hms/src/security/thrift_sasl.rs +++ b/crates/sail-catalog-hms/src/security/thrift_sasl.rs @@ -9,10 +9,10 @@ use sail_catalog::error::{CatalogError, CatalogResult}; const MAX_SASL_FRAME_SIZE: usize = 0x00FF_FFFF; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, Interest, Ready}; use tokio::net::tcp; +use volo::net::Address; use volo::net::conn::{OwnedReadHalf, OwnedWriteHalf}; use volo::net::dial::{DefaultMakeTransport, MakeTransport}; use volo::net::ext::AsyncExt; -use volo::net::Address; use super::gssapi::{GssapiFrameProtector, GssapiSession, SaslQop}; @@ -98,7 +98,7 @@ impl MakeTransport for KerberosMakeTransport { return Err(io::Error::new( io::ErrorKind::Unsupported, "Kerberos HMS transport only supports TCP connections", - )) + )); } }; @@ -330,11 +330,11 @@ impl AsyncWrite for SaslWriteHalf { return std::task::Poll::Ready(Err(io::Error::new( io::ErrorKind::WriteZero, "failed to write SASL-framed HMS payload", - ))) + ))); } std::task::Poll::Ready(Ok(n)) => written += n, std::task::Poll::Ready(Err(error)) => { - return std::task::Poll::Ready(Err(error)) + return std::task::Poll::Ready(Err(error)); } std::task::Poll::Pending => { self.frame = Some((frame, written)); @@ -413,7 +413,7 @@ where other => { return Err(CatalogError::External(format!( "Expected Thrift SASL OK or COMPLETE from HMS, got {other:?}" - ))) + ))); } } @@ -528,12 +528,12 @@ mod tests { use std::sync::{Arc, Mutex}; - use tokio::io::{duplex, AsyncRead, AsyncWrite}; - use tokio::time::{timeout, Duration}; + use tokio::io::{AsyncRead, AsyncWrite, duplex}; + use tokio::time::{Duration, timeout}; use super::{ - perform_thrift_sasl_handshake, AsyncReadExt, AsyncWriteExt, CatalogError, CatalogResult, - NegotiationStatus, SaslClientSession, + AsyncReadExt, AsyncWriteExt, CatalogError, CatalogResult, NegotiationStatus, + SaslClientSession, perform_thrift_sasl_handshake, }; #[derive(Debug, Clone)] diff --git a/crates/sail-catalog-hms/tests/common/mod.rs b/crates/sail-catalog-hms/tests/common/mod.rs index 5715be1a3..cd74138eb 100644 --- a/crates/sail-catalog-hms/tests/common/mod.rs +++ b/crates/sail-catalog-hms/tests/common/mod.rs @@ -1,10 +1,9 @@ #![allow(dead_code)] #![expect(clippy::expect_used, clippy::panic)] -use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::{Arc, OnceLock}; +use std::sync::OnceLock; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use arrow::datatypes::DataType; @@ -20,7 +19,7 @@ use testcontainers::runners::{AsyncBuilder, AsyncRunner}; use testcontainers::{ContainerAsync, GenericBuildableImage, GenericImage, Image, ImageExt}; use tokio::net::TcpStream; use tokio::process::Command; -use tokio::sync::{Mutex, OwnedMutexGuard}; +use tokio::sync::Mutex; const KERBEROS_REALM: &str = "SAIL.TEST"; const KDC_HOSTNAME: &str = "sail-kerberos-kdc"; @@ -31,6 +30,13 @@ const HIVE_SITE_XML_PATH: &str = "/opt/hive/conf/hive-site.xml"; const CORE_SITE_XML_PATH: &str = "/opt/hadoop/etc/hadoop/core-site.xml"; const HIVE_METASTORE_PORT: u16 = 9083; const KDC_PORT: u16 = 88; +const KRB_SUBPROCESS_ENV: &str = "SAIL_HMS_KRB_SUBPROCESS"; +const KRB_TEST_NAME_ENV: &str = "SAIL_HMS_KRB_TEST_NAME"; +const KRB_CANONICAL_HOST_ENV: &str = "SAIL_HMS_KRB_CANONICAL_HOST"; +const KRB_HMS_PORT_ENV: &str = "SAIL_HMS_KRB_HMS_PORT"; +const KRB_SERVICE_PRINCIPAL_ENV: &str = "SAIL_HMS_KRB_SERVICE_PRINCIPAL"; +const KRB_CLIENT_PRINCIPAL_ENV: &str = "SAIL_HMS_KRB_CLIENT_PRINCIPAL"; +const KRB_CLIENT_KEYTAB_ENV: &str = "SAIL_HMS_KRB_CLIENT_KEYTAB"; pub struct HmsTestContext { pub catalog: HmsCatalogProvider, @@ -45,39 +51,6 @@ pub struct HmsDatabaseContext { pub struct KerberosHmsTestContext { pub catalog: HmsCatalogProvider, - _env_guard: ProcessEnvGuard, - _lock: OwnedMutexGuard<()>, -} - -struct ProcessEnvGuard { - previous: Vec<(String, Option)>, -} - -impl ProcessEnvGuard { - fn new() -> Self { - Self { - previous: Vec::new(), - } - } - - fn set(&mut self, name: &str, value: impl Into) { - if !self.previous.iter().any(|(key, _)| key == name) { - self.previous - .push((name.to_string(), std::env::var_os(name))); - } - std::env::set_var(name, value.into()); - } -} - -impl Drop for ProcessEnvGuard { - fn drop(&mut self) { - for (name, value) in self.previous.drain(..).rev() { - match value { - Some(value) => std::env::set_var(name, value), - None => std::env::remove_var(name), - } - } - } } pub fn simple_database_options() -> CreateDatabaseOptions { @@ -228,37 +201,68 @@ async fn shared_hms_container() -> &'static SharedHmsContainer { SHARED_HMS.get().unwrap() } -pub async fn setup_kerberos_hms_catalog(test_name: &str) -> KerberosHmsTestContext { - setup_kerberos_hms_catalog_inner(test_name, true).await +pub fn is_kerberos_hms_subprocess() -> bool { + std::env::var_os(KRB_SUBPROCESS_ENV).is_some() } -pub async fn setup_kerberos_hms_catalog_without_kinit(test_name: &str) -> KerberosHmsTestContext { - setup_kerberos_hms_catalog_inner(test_name, false).await +pub async fn run_kerberos_hms_catalog_test_in_subprocess(test_name: &str, helper_test: &str) { + let shared = shared_kerberos_infrastructure().await; + let ticket_cache_path = shared.temp_dir.path().join(format!("krb5cc-{test_name}")); + let ticket_cache = format!("FILE:{}", ticket_cache_path.display()); + let current_exe = + std::env::current_exe().unwrap_or_else(|error| panic!("get current test binary: {error}")); + + let output = Command::new(current_exe) + .arg("--ignored") + .arg("--exact") + .arg(helper_test) + .env(KRB_SUBPROCESS_ENV, "1") + .env(KRB_TEST_NAME_ENV, test_name) + .env("KRB5_CONFIG", &shared.host_krb5_conf_path) + .env("KRB5CCNAME", ticket_cache) + .env("SAIL_HMS_KRB_TRACE", "1") + .env(KRB_CANONICAL_HOST_ENV, &shared.canonical_host) + .env(KRB_HMS_PORT_ENV, shared.hms_port.to_string()) + .env(KRB_SERVICE_PRINCIPAL_ENV, &shared.service_principal) + .env(KRB_CLIENT_PRINCIPAL_ENV, &shared.client_principal) + .env(KRB_CLIENT_KEYTAB_ENV, &shared.client_keytab_path) + .output() + .await + .unwrap_or_else(|error| panic!("run Kerberos HMS subprocess test {helper_test}: {error}")); + + if !output.status.success() { + panic!( + "Kerberos HMS subprocess test {helper_test} failed: status={:?}, stdout={}, stderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } } -async fn setup_kerberos_hms_catalog_inner( - test_name: &str, +pub async fn setup_kerberos_hms_catalog_from_subprocess_env( perform_kinit: bool, ) -> KerberosHmsTestContext { - let lock = kerberos_test_lock().lock_owned().await; - let shared = shared_kerberos_infrastructure().await; - - let mut env_guard = ProcessEnvGuard::new(); - let ticket_cache_path = shared.temp_dir.path().join(format!("krb5cc-{test_name}")); - let ticket_cache_str = format!("FILE:{}", ticket_cache_path.display()); - env_guard.set("KRB5CCNAME", ticket_cache_str); + let test_name = required_env(KRB_TEST_NAME_ENV); + let canonical_host = required_env(KRB_CANONICAL_HOST_ENV); + let hms_port = required_env(KRB_HMS_PORT_ENV) + .parse::() + .unwrap_or_else(|error| panic!("parse {KRB_HMS_PORT_ENV}: {error}")); + let service_principal = required_env(KRB_SERVICE_PRINCIPAL_ENV); + let client_principal = required_env(KRB_CLIENT_PRINCIPAL_ENV); + let client_keytab_path = PathBuf::from(required_env(KRB_CLIENT_KEYTAB_ENV)); if perform_kinit { - run_kinit(&shared.client_keytab_path, &shared.client_principal).await; + run_kinit(&client_keytab_path, &client_principal).await; } let catalog = HmsCatalogProvider::new( - test_name.to_string(), + test_name, HmsCatalogConfig { - uris: vec![format!("{}:{}", shared.canonical_host, shared.hms_port)], + uris: vec![format!("{canonical_host}:{hms_port}")], thrift_transport: None, auth: Some("kerberos".to_string()), - kerberos_service_principal: Some(shared.service_principal.clone()), + kerberos_service_principal: Some(service_principal), min_sasl_qop: None, connect_timeout_secs: None, }, @@ -270,11 +274,11 @@ async fn setup_kerberos_hms_catalog_inner( wait_until_ready(&catalog, 240, "Kerberos Hive Metastore").await; } - KerberosHmsTestContext { - catalog, - _env_guard: env_guard, - _lock: lock, - } + KerberosHmsTestContext { catalog } +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|error| panic!("read {name}: {error}")) } struct SharedKerberosInfrastructure { @@ -424,12 +428,6 @@ async fn shared_kerberos_infrastructure() -> &'static SharedKerberosInfrastructu .unwrap_or_else(|error| panic!("get HMS host port: {error}")); wait_for_tcp_port(&canonical_host, hms_port, 240, "Kerberos Hive Metastore").await; - // Set Kerberos env vars permanently for the process (not per-test). - // KRB5_CONFIG must be set before any GSSAPI call so the library can - // locate the KDC. SAIL_HMS_KRB_TRACE enables readiness diagnostics. - std::env::set_var("KRB5_CONFIG", &host_krb5_conf_path); - std::env::set_var("SAIL_HMS_KRB_TRACE", "1"); - let _ = SHARED_KRB.set(SharedKerberosInfrastructure { canonical_host, hms_port, @@ -451,12 +449,6 @@ fn runtime_handle() -> RuntimeHandle { ) } -fn kerberos_test_lock() -> Arc> { - static LOCK: OnceLock>> = OnceLock::new(); - LOCK.get_or_init(|| Arc::new(tokio::sync::Mutex::new(()))) - .clone() -} - async fn wait_until_ready(catalog: &HmsCatalogProvider, attempts: usize, service_name: &str) { let mut last_error = None; for attempt in 0..attempts { diff --git a/crates/sail-catalog-hms/tests/database_tests.rs b/crates/sail-catalog-hms/tests/database_tests.rs index ed3533312..554673ae3 100644 --- a/crates/sail-catalog-hms/tests/database_tests.rs +++ b/crates/sail-catalog-hms/tests/database_tests.rs @@ -29,9 +29,11 @@ async fn test_create_get_drop_database() { ); let databases = catalog.list_databases(None).await.unwrap(); - assert!(databases - .iter() - .any(|db| db.database == vec!["test_create_get_drop_database".to_string()])); + assert!( + databases + .iter() + .any(|db| db.database == vec!["test_create_get_drop_database".to_string()]) + ); catalog .drop_database( diff --git a/crates/sail-catalog-hms/tests/kerberos_integration_tests.rs b/crates/sail-catalog-hms/tests/kerberos_integration_tests.rs index 68c8d1c70..e1d5d4697 100644 --- a/crates/sail-catalog-hms/tests/kerberos_integration_tests.rs +++ b/crates/sail-catalog-hms/tests/kerberos_integration_tests.rs @@ -6,15 +6,32 @@ use arrow::datatypes::DataType; use sail_catalog::provider::{CatalogProvider, DropDatabaseOptions, DropTableOptions, Namespace}; use crate::common::{ - col, setup_kerberos_hms_catalog, setup_kerberos_hms_catalog_without_kinit, + KerberosHmsTestContext, col, is_kerberos_hms_subprocess, + run_kerberos_hms_catalog_test_in_subprocess, setup_kerberos_hms_catalog_from_subprocess_env, simple_database_options, simple_table_options, }; #[tokio::test] #[ignore = "requires the Kerberos HMS testcontainers harness"] async fn test_kerberos_hms_database_and_table_round_trip() { - let context = - setup_kerberos_hms_catalog("test_kerberos_hms_database_and_table_round_trip").await; + run_kerberos_hms_catalog_test_in_subprocess( + "test_kerberos_hms_database_and_table_round_trip", + "kerberos_hms_database_and_table_round_trip_subprocess", + ) + .await; +} + +#[tokio::test] +#[ignore = "subprocess helper for the Kerberos HMS testcontainers harness"] +async fn kerberos_hms_database_and_table_round_trip_subprocess() { + if !is_kerberos_hms_subprocess() { + return; + } + let context = setup_kerberos_hms_catalog_from_subprocess_env(true).await; + kerberos_hms_database_and_table_round_trip(context).await; +} + +async fn kerberos_hms_database_and_table_round_trip(context: KerberosHmsTestContext) { let catalog = &context.catalog; let db_name = "kerberos_hms_db".to_string(); let namespace = Namespace::try_from(vec![db_name.clone()]).unwrap(); @@ -69,9 +86,20 @@ async fn test_kerberos_hms_database_and_table_round_trip() { #[tokio::test] #[ignore = "requires the Kerberos HMS testcontainers harness"] async fn test_kerberos_hms_rejects_missing_credentials() { - let context = - setup_kerberos_hms_catalog_without_kinit("test_kerberos_hms_rejects_missing_credentials") - .await; + run_kerberos_hms_catalog_test_in_subprocess( + "test_kerberos_hms_rejects_missing_credentials", + "kerberos_hms_rejects_missing_credentials_subprocess", + ) + .await; +} + +#[tokio::test] +#[ignore = "subprocess helper for the Kerberos HMS testcontainers harness"] +async fn kerberos_hms_rejects_missing_credentials_subprocess() { + if !is_kerberos_hms_subprocess() { + return; + } + let context = setup_kerberos_hms_catalog_from_subprocess_env(false).await; let error = context.catalog.list_databases(None).await.unwrap_err(); let message = error.to_string().to_ascii_lowercase(); diff --git a/crates/sail-catalog-hms/tests/table_tests.rs b/crates/sail-catalog-hms/tests/table_tests.rs index 355828628..643fb9439 100644 --- a/crates/sail-catalog-hms/tests/table_tests.rs +++ b/crates/sail-catalog-hms/tests/table_tests.rs @@ -65,7 +65,6 @@ async fn test_supported_formats_round_trip() { ("parquet_items", "parquet", "parquet"), ("csv_items", "csv", "csv"), ("delta_items", "delta", "delta"), - ("deltalake_items", "deltalake", "delta"), ("textfile_items", "textfile", "textfile"), ("json_items", "json", "json"), ("orc_items", "orc", "orc"), diff --git a/crates/sail-catalog-iceberg/Cargo.toml b/crates/sail-catalog-iceberg/Cargo.toml index 2d9d39e77..a60737dc9 100644 --- a/crates/sail-catalog-iceberg/Cargo.toml +++ b/crates/sail-catalog-iceberg/Cargo.toml @@ -24,6 +24,7 @@ sail-common = { path = "../sail-common" } sail-common-datafusion = { path = "../sail-common-datafusion" } sail-iceberg = { path = "../sail-iceberg" } +arrow = { workspace = true } async-trait = { workspace = true } reqwest = { workspace = true } url = { workspace = true } @@ -31,11 +32,16 @@ percent-encoding = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true } +thiserror = { workspace = true } uuid = { workspace = true } tokio = { workspace = true } +log = { workspace = true } [dev-dependencies] tokio = { workspace = true } wiremock = { workspace = true } testcontainers = { workspace = true } -arrow = { workspace = true } +tempfile = { workspace = true } + +[build-dependencies] +sail-build-scripts = { path = "../sail-build-scripts" } diff --git a/crates/sail-catalog-iceberg/build.rs b/crates/sail-catalog-iceberg/build.rs new file mode 100644 index 000000000..b53f81f4d --- /dev/null +++ b/crates/sail-catalog-iceberg/build.rs @@ -0,0 +1,30 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sail_build_scripts::openapi::generator::{OpenApiConfig, generate_openapi_client}; + +fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed=build.rs"); + + generate_openapi_client( + "spec/iceberg-rest-catalog.yaml", + std::path::Path::new(&std::env::var("OUT_DIR")?).join("iceberg_rest_catalog_gen.rs"), + OpenApiConfig { + excluded_operations: ["getToken".to_owned()].into_iter().collect(), + excluded_schemas: ["ReportMetricsRequest".to_owned(), "Snapshot".to_owned()] + .into_iter() + .collect(), + }, + )?; + + Ok(()) +} diff --git a/crates/sail-catalog-iceberg/spec/generate-client.sh b/crates/sail-catalog-iceberg/spec/generate-client.sh deleted file mode 100755 index 59316a0d9..000000000 --- a/crates/sail-catalog-iceberg/spec/generate-client.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -# Determine script directory and repository root -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" - -cd "$REPO_ROOT" - -echo "==> Generating Iceberg REST Catalog client..." - -rm -rf crates/sail-catalog-iceberg/src/generated_rest_temp - -openapi-generator generate \ - --generator-name rust \ - --config crates/sail-catalog-iceberg/spec/openapi-generator-config.yaml \ - --input-spec crates/sail-catalog-iceberg/spec/iceberg-rest-catalog.yaml \ - --output crates/sail-catalog-iceberg/src/generated_rest_temp \ - --skip-validate-spec \ - --schema-mappings Type=sail_iceberg::spec::Type,StructType=sail_iceberg::spec::StructType,ListType=sail_iceberg::spec::ListType,MapType=sail_iceberg::spec::MapType,StructField=sail_iceberg::spec::NestedFieldRef - -echo "==> Flattening generated code to src/..." - -# Remove old generated directories and apis/ and models/ directly to src/ -rm -rf crates/sail-catalog-iceberg/src/apis -rm -rf crates/sail-catalog-iceberg/src/models -cp -r crates/sail-catalog-iceberg/src/generated_rest_temp/src/apis crates/sail-catalog-iceberg/src/ -cp -r crates/sail-catalog-iceberg/src/generated_rest_temp/src/models crates/sail-catalog-iceberg/src/ - -rm -rf crates/sail-catalog-iceberg/src/generated_rest_temp - -echo "==> Running cargo fmt..." -cargo +nightly fmt \ No newline at end of file diff --git a/crates/sail-catalog-iceberg/spec/openapi-generator-config.yaml b/crates/sail-catalog-iceberg/spec/openapi-generator-config.yaml deleted file mode 100644 index 76ef6940d..000000000 --- a/crates/sail-catalog-iceberg/spec/openapi-generator-config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -additionalProperties: - library: reqwest-trait - supportAsync: true - topLevelApiClient: true - -globalProperties: - apiDocs: false - modelDocs: false diff --git a/crates/sail-catalog-iceberg/src/adapter.rs b/crates/sail-catalog-iceberg/src/adapter.rs new file mode 100644 index 000000000..a503aa05f --- /dev/null +++ b/crates/sail-catalog-iceberg/src/adapter.rs @@ -0,0 +1,59 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sail_catalog::error::CatalogError; +use sail_catalog::provider::Namespace; +use sail_iceberg::spec::Schema; +use sail_iceberg::spec::types::Type; + +use crate::r#gen; + +impl From> for r#gen::Namespace { + fn from(value: Vec) -> Self { + Self(value) + } +} + +impl From for Vec { + fn from(value: r#gen::Namespace) -> Self { + value.0 + } +} + +impl From for r#gen::Namespace { + fn from(value: Namespace) -> Self { + let value: Vec = value.into(); + value.into() + } +} + +impl TryFrom for r#gen::Schema { + type Error = CatalogError; + + fn try_from(value: Schema) -> Result { + serde_json::to_value(value) + .and_then(serde_json::from_value) + .map_err(|e| CatalogError::External(format!("failed to convert Iceberg schema: {e}"))) + } +} + +impl TryFrom for Type { + type Error = CatalogError; + + fn try_from(value: r#gen::Type) -> Result { + serde_json::to_value(value) + .and_then(serde_json::from_value) + .map_err(|e| { + CatalogError::External(format!("failed to convert Iceberg type from REST: {e}")) + }) + } +} diff --git a/crates/sail-catalog-iceberg/src/apis/catalog_api_api.rs b/crates/sail-catalog-iceberg/src/apis/catalog_api_api.rs deleted file mode 100644 index 6556314b7..000000000 --- a/crates/sail-catalog-iceberg/src/apis/catalog_api_api.rs +++ /dev/null @@ -1,2510 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use std::sync::Arc; - -use async_trait::async_trait; -use reqwest; -use serde::de::Error as _; -use serde::{Deserialize, Serialize}; - -use super::{configuration, Error}; -use crate::apis::{ContentType, ResponseContent}; -use crate::models; - -#[async_trait] -pub trait CatalogApiApi: Send + Sync { - /// DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan/{plan_id} - /// - /// Cancels scan planning for a plan-id. This notifies the service that it can release resources held for the scan. Clients should cancel scans that are no longer needed, either while the plan-id returns a \"submitted\" status or while there are remaining plan tasks that have not been fetched. Cancellation is not necessary when - Scan tasks for each plan task have been fetched using fetchScanTasks - A plan-id has produced a \"failed\" or \"cancelled\" status from planTableScan or fetchPlanningResult - async fn cancel_planning<'namespace, 'table, 'plan_id, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - plan_id: &'plan_id str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// POST /v1/{prefix}/transactions/commit - /// - /// - async fn commit_transaction<'commit_transaction_request, 'prefix>( - &self, - commit_transaction_request: models::CommitTransactionRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// POST /v1/{prefix}/namespaces - /// - /// Create a namespace, with an optional set of properties. The server might also add properties, such as `last_modified_time` etc. - async fn create_namespace<'create_namespace_request, 'prefix>( - &self, - create_namespace_request: models::CreateNamespaceRequest, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// POST /v1/{prefix}/namespaces/{namespace}/tables - /// - /// Create a table or start a create transaction, like atomic CTAS. If `stage-create` is false, the table is created immediately. If `stage-create` is true, the table is not created, but table metadata is initialized and returned. The service should prepare as needed for a commit to the table commit endpoint to complete the create transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, the client sends all create and subsequent changes to the table commit route. Changes from the table create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial table state. - async fn create_table< - 'namespace, - 'create_table_request, - 'x_iceberg_access_delegation, - 'prefix, - >( - &self, - namespace: &'namespace str, - create_table_request: models::CreateTableRequest, - x_iceberg_access_delegation: Option<&'x_iceberg_access_delegation str>, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// POST /v1/{prefix}/namespaces/{namespace}/views - /// - /// Create a view in the given namespace. - async fn create_view<'namespace, 'create_view_request, 'prefix>( - &self, - namespace: &'namespace str, - create_view_request: models::CreateViewRequest, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// DELETE /v1/{prefix}/namespaces/{namespace} - /// - /// - async fn drop_namespace<'namespace, 'prefix>( - &self, - namespace: &'namespace str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table} - /// - /// Remove a table from the catalog - async fn drop_table<'namespace, 'table, 'purge_requested, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - purge_requested: Option, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// DELETE /v1/{prefix}/namespaces/{namespace}/views/{view} - /// - /// Remove a view from the catalog - async fn drop_view<'namespace, 'view, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan/{plan_id} - /// - /// Fetches the result of scan planning for a plan-id. Responses must include a valid status - When \"completed\" the planning operation has produced plan-tasks and file-scan-tasks that must be returned in the response - When \"submitted\" the planning operation has not completed; the client should wait to call this endpoint again to fetch a completed response - When \"failed\" the response must be a valid error response - When \"cancelled\" the plan-id is invalid and should be discarded The response for a \"completed\" planning operation includes two types of tasks (file scan tasks and plan tasks) and both may be included in the response. Tasks must not be included for any other response status. - async fn fetch_planning_result<'namespace, 'table, 'plan_id, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - plan_id: &'plan_id str, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/tasks - /// - /// Fetches result tasks for a plan task. - async fn fetch_scan_tasks<'namespace, 'table, 'prefix, 'fetch_scan_tasks_request>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - fetch_scan_tasks_request: Option, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces - /// - /// List all namespaces at a certain level, optionally starting from a given parent namespace. If table accounting.tax.paid.info exists, using 'SELECT NAMESPACE IN accounting' would translate into `GET /namespaces?parent=accounting` and must return a namespace, [\"accounting\", \"tax\"] only. Using 'SELECT NAMESPACE IN accounting.tax' would translate into `GET /namespaces?parent=accounting%1Ftax` and must return a namespace, [\"accounting\", \"tax\", \"paid\"]. If `parent` is not provided, all top-level namespaces should be listed. - async fn list_namespaces<'page_token, 'page_size, 'parent, 'prefix>( - &self, - page_token: Option<&'page_token str>, - page_size: Option, - parent: Option<&'parent str>, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces/{namespace}/tables - /// - /// Return all table identifiers under this namespace - async fn list_tables<'namespace, 'page_token, 'page_size, 'prefix>( - &self, - namespace: &'namespace str, - page_token: Option<&'page_token str>, - page_size: Option, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces/{namespace}/views - /// - /// Return all view identifiers under this namespace - async fn list_views<'namespace, 'page_token, 'page_size, 'prefix>( - &self, - namespace: &'namespace str, - page_token: Option<&'page_token str>, - page_size: Option, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials - /// - /// Load vended credentials for a table from the catalog. - async fn load_credentials<'namespace, 'table, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces/{namespace} - /// - /// Return all stored metadata properties for a given namespace - async fn load_namespace_metadata<'namespace, 'prefix>( - &self, - namespace: &'namespace str, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces/{namespace}/tables/{table} - /// - /// Load a table from the catalog. The response contains both configuration and table metadata. The configuration, if non-empty is used as additional configuration for the table that overrides catalog configuration. For example, this configuration may change the FileIO implementation to be used for the table. The response also contains the table's full metadata, matching the table metadata JSON file. The catalog configuration may contain credentials that should be used for subsequent requests for the table. The configuration key \"token\" is used to pass an access token to be used as a bearer token for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, \"urn:ietf:params:oauth:token-type:jwt=\". - async fn load_table< - 'namespace, - 'table, - 'x_iceberg_access_delegation, - 'if_none_match, - 'snapshots, - 'prefix, - >( - &self, - namespace: &'namespace str, - table: &'table str, - x_iceberg_access_delegation: Option<&'x_iceberg_access_delegation str>, - if_none_match: Option<&'if_none_match str>, - snapshots: Option<&'snapshots str>, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// GET /v1/{prefix}/namespaces/{namespace}/views/{view} - /// - /// Load a view from the catalog. The response contains both configuration and view metadata. The configuration, if non-empty is used as additional configuration for the view that overrides catalog configuration. The response also contains the view's full metadata, matching the view metadata JSON file. The catalog configuration may contain credentials that should be used for subsequent requests for the view. The configuration key \"token\" is used to pass an access token to be used as a bearer token for view requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, \"urn:ietf:params:oauth:token-type:jwt=\". - async fn load_view<'namespace, 'view, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// HEAD /v1/{prefix}/namespaces/{namespace} - /// - /// Check if a namespace exists. The response does not contain a body. - async fn namespace_exists<'namespace, 'prefix>( - &self, - namespace: &'namespace str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan - /// - /// Submits a scan for server-side planning. Point-in-time scans are planned by passing snapshot-id to identify the table snapshot to scan. Incremental scans are planned by passing both start-snapshot-id and end-snapshot-id. Requests that include both point in time config properties and incremental config properties are invalid. If the request does not include either incremental or point-in-time config properties, scan planning should produce a point-in-time scan of the latest snapshot in the table's main branch. Responses must include a valid status listed below. A \"cancelled\" status is considered invalid for this endpoint. - When \"completed\" the planning operation has produced plan tasks and file scan tasks that must be returned in the response (not fetched later by calling fetchPlanningResult) - When \"submitted\" the response must include a plan-id used to poll fetchPlanningResult to fetch the planning result when it is ready - When \"failed\" the response must be a valid error response The response for a \"completed\" planning operation includes two types of tasks (file scan tasks and plan tasks) and both may be included in the response. Tasks must not be included for any other response status. Responses that include a plan-id indicate that the service is holding state or performing work for the client. - Clients should use the plan-id to fetch results from fetchPlanningResult when the response status is \"submitted\" - Clients should inform the service if planning results are no longer needed by calling cancelPlanning. Cancellation is not necessary after fetchScanTasks has been used to fetch scan tasks for each plan task. - async fn plan_table_scan<'namespace, 'table, 'prefix, 'plan_table_scan_request>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - plan_table_scan_request: Option, - ) -> Result>; - - /// POST /v1/{prefix}/namespaces/{namespace}/register - /// - /// Register a table using given metadata file location. - async fn register_table<'namespace, 'register_table_request, 'prefix>( - &self, - namespace: &'namespace str, - register_table_request: models::RegisterTableRequest, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// POST /v1/{prefix}/tables/rename - /// - /// Rename a table from one identifier to another. It's valid to move a table across namespaces, but the server implementation is not required to support it. - async fn rename_table<'rename_table_request, 'prefix>( - &self, - rename_table_request: models::RenameTableRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// POST /v1/{prefix}/views/rename - /// - /// Rename a view from one identifier to another. It's valid to move a view across namespaces, but the server implementation is not required to support it. - async fn rename_view<'rename_table_request, 'prefix>( - &self, - rename_table_request: models::RenameTableRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// POST /v1/{prefix}/namespaces/{namespace}/views/{view} - /// - /// Commit updates to a view. - async fn replace_view<'namespace, 'view, 'commit_view_request, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - commit_view_request: models::CommitViewRequest, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics - /// - /// - async fn report_metrics<'namespace, 'table, 'report_metrics_request, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - report_metrics_request: models::ReportMetricsRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table} - /// - /// Check if a table exists within a given namespace. The response does not contain a body. - async fn table_exists<'namespace, 'table, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; - - /// POST /v1/{prefix}/namespaces/{namespace}/properties - /// - /// Set and/or remove properties on a namespace. The request body specifies a list of properties to remove and a map of key value pairs to update. Properties that are not in the request are not modified or removed by this call. Server implementations are not required to support namespace properties. - async fn update_properties<'namespace, 'update_namespace_properties_request, 'prefix>( - &self, - namespace: &'namespace str, - update_namespace_properties_request: models::UpdateNamespacePropertiesRequest, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// POST /v1/{prefix}/namespaces/{namespace}/tables/{table} - /// - /// Commit updates to a table. Commits have two parts, requirements and updates. Requirements are assertions that will be validated before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a named ref's snapshot ID has a certain value. Server implementations are required to fail with a 400 status code if any unknown updates or requirements are received. Updates are changes to make to table metadata. For example, after asserting that the current main ref is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new snapshot id. Create table transactions that are started by createTable with `stage-create` set to true are committed using this route. Transactions should include all changes to the table, including table initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is used to ensure that the table was not created concurrently. - async fn update_table<'namespace, 'table, 'commit_table_request, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - commit_table_request: models::CommitTableRequest, - prefix: Option<&'prefix str>, - ) -> Result>; - - /// HEAD /v1/{prefix}/namespaces/{namespace}/views/{view} - /// - /// Check if a view exists within a given namespace. This request does not return a response body. - async fn view_exists<'namespace, 'view, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error>; -} - -pub struct CatalogApiApiClient { - configuration: Arc, -} - -impl CatalogApiApiClient { - pub fn new(configuration: Arc) -> Self { - Self { configuration } - } -} - -#[async_trait] -impl CatalogApiApi for CatalogApiApiClient { - /// Cancels scan planning for a plan-id. This notifies the service that it can release resources held for the scan. Clients should cancel scans that are no longer needed, either while the plan-id returns a \"submitted\" status or while there are remaining plan tasks that have not been fetched. Cancellation is not necessary when - Scan tasks for each plan task have been fetched using fetchScanTasks - A plan-id has produced a \"failed\" or \"cancelled\" status from planTableScan or fetchPlanningResult - async fn cancel_planning<'namespace, 'table, 'plan_id, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - plan_id: &'plan_id str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}/plan/{plan_id}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table), - plan_id = crate::apis::urlencode(plan_id) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::DELETE, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - async fn commit_transaction<'commit_transaction_request, 'prefix>( - &self, - commit_transaction_request: models::CommitTransactionRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/transactions/commit", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default() - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&commit_transaction_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Create a namespace, with an optional set of properties. The server might also add properties, such as `last_modified_time` etc. - async fn create_namespace<'create_namespace_request, 'prefix>( - &self, - create_namespace_request: models::CreateNamespaceRequest, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default() - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&create_namespace_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::CreateNamespaceResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::CreateNamespaceResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Create a table or start a create transaction, like atomic CTAS. If `stage-create` is false, the table is created immediately. If `stage-create` is true, the table is not created, but table metadata is initialized and returned. The service should prepare as needed for a commit to the table commit endpoint to complete the create transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, the client sends all create and subsequent changes to the table commit route. Changes from the table create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial table state. - async fn create_table< - 'namespace, - 'create_table_request, - 'x_iceberg_access_delegation, - 'prefix, - >( - &self, - namespace: &'namespace str, - create_table_request: models::CreateTableRequest, - x_iceberg_access_delegation: Option<&'x_iceberg_access_delegation str>, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(local_var_param_value) = x_iceberg_access_delegation { - local_var_req_builder = local_var_req_builder.header( - "X-Iceberg-Access-Delegation", - local_var_param_value.to_string(), - ); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&create_table_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadTableResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadTableResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Create a view in the given namespace. - async fn create_view<'namespace, 'create_view_request, 'prefix>( - &self, - namespace: &'namespace str, - create_view_request: models::CreateViewRequest, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/views", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&create_view_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadViewResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadViewResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - async fn drop_namespace<'namespace, 'prefix>( - &self, - namespace: &'namespace str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::DELETE, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Remove a table from the catalog - async fn drop_table<'namespace, 'table, 'purge_requested, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - purge_requested: Option, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::DELETE, local_var_uri_str.as_str()); - - if let Some(ref param_value) = purge_requested { - local_var_req_builder = - local_var_req_builder.query(&[("purgeRequested", ¶m_value.to_string())]); - } - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Remove a view from the catalog - async fn drop_view<'namespace, 'view, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/views/{view}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - view = crate::apis::urlencode(view) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::DELETE, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Fetches the result of scan planning for a plan-id. Responses must include a valid status - When \"completed\" the planning operation has produced plan-tasks and file-scan-tasks that must be returned in the response - When \"submitted\" the planning operation has not completed; the client should wait to call this endpoint again to fetch a completed response - When \"failed\" the response must be a valid error response - When \"cancelled\" the plan-id is invalid and should be discarded The response for a \"completed\" planning operation includes two types of tasks (file scan tasks and plan tasks) and both may be included in the response. Tasks must not be included for any other response status. - async fn fetch_planning_result<'namespace, 'table, 'plan_id, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - plan_id: &'plan_id str, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}/plan/{plan_id}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table), - plan_id = crate::apis::urlencode(plan_id) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::FetchPlanningResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::FetchPlanningResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Fetches result tasks for a plan task. - async fn fetch_scan_tasks<'namespace, 'table, 'prefix, 'fetch_scan_tasks_request>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - fetch_scan_tasks_request: Option, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}/tasks", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&fetch_scan_tasks_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::FetchScanTasksResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::FetchScanTasksResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// List all namespaces at a certain level, optionally starting from a given parent namespace. If table accounting.tax.paid.info exists, using 'SELECT NAMESPACE IN accounting' would translate into `GET /namespaces?parent=accounting` and must return a namespace, [\"accounting\", \"tax\"] only. Using 'SELECT NAMESPACE IN accounting.tax' would translate into `GET /namespaces?parent=accounting%1Ftax` and must return a namespace, [\"accounting\", \"tax\", \"paid\"]. If `parent` is not provided, all top-level namespaces should be listed. - async fn list_namespaces<'page_token, 'page_size, 'parent, 'prefix>( - &self, - page_token: Option<&'page_token str>, - page_size: Option, - parent: Option<&'parent str>, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default() - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref param_value) = page_token { - local_var_req_builder = - local_var_req_builder.query(&[("pageToken", ¶m_value.to_string())]); - } - if let Some(ref param_value) = page_size { - local_var_req_builder = - local_var_req_builder.query(&[("pageSize", ¶m_value.to_string())]); - } - if let Some(ref param_value) = parent { - local_var_req_builder = - local_var_req_builder.query(&[("parent", ¶m_value.to_string())]); - } - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ListNamespacesResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::ListNamespacesResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Return all table identifiers under this namespace - async fn list_tables<'namespace, 'page_token, 'page_size, 'prefix>( - &self, - namespace: &'namespace str, - page_token: Option<&'page_token str>, - page_size: Option, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref param_value) = page_token { - local_var_req_builder = - local_var_req_builder.query(&[("pageToken", ¶m_value.to_string())]); - } - if let Some(ref param_value) = page_size { - local_var_req_builder = - local_var_req_builder.query(&[("pageSize", ¶m_value.to_string())]); - } - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ListTablesResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::ListTablesResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Return all view identifiers under this namespace - async fn list_views<'namespace, 'page_token, 'page_size, 'prefix>( - &self, - namespace: &'namespace str, - page_token: Option<&'page_token str>, - page_size: Option, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/views", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref param_value) = page_token { - local_var_req_builder = - local_var_req_builder.query(&[("pageToken", ¶m_value.to_string())]); - } - if let Some(ref param_value) = page_size { - local_var_req_builder = - local_var_req_builder.query(&[("pageSize", ¶m_value.to_string())]); - } - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ListTablesResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::ListTablesResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Load vended credentials for a table from the catalog. - async fn load_credentials<'namespace, 'table, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}/credentials", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadCredentialsResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadCredentialsResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Return all stored metadata properties for a given namespace - async fn load_namespace_metadata<'namespace, 'prefix>( - &self, - namespace: &'namespace str, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::GetNamespaceResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::GetNamespaceResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Load a table from the catalog. The response contains both configuration and table metadata. The configuration, if non-empty is used as additional configuration for the table that overrides catalog configuration. For example, this configuration may change the FileIO implementation to be used for the table. The response also contains the table's full metadata, matching the table metadata JSON file. The catalog configuration may contain credentials that should be used for subsequent requests for the table. The configuration key \"token\" is used to pass an access token to be used as a bearer token for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, \"urn:ietf:params:oauth:token-type:jwt=\". - async fn load_table< - 'namespace, - 'table, - 'x_iceberg_access_delegation, - 'if_none_match, - 'snapshots, - 'prefix, - >( - &self, - namespace: &'namespace str, - table: &'table str, - x_iceberg_access_delegation: Option<&'x_iceberg_access_delegation str>, - if_none_match: Option<&'if_none_match str>, - snapshots: Option<&'snapshots str>, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref param_value) = snapshots { - local_var_req_builder = - local_var_req_builder.query(&[("snapshots", ¶m_value.to_string())]); - } - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(local_var_param_value) = x_iceberg_access_delegation { - local_var_req_builder = local_var_req_builder.header( - "X-Iceberg-Access-Delegation", - local_var_param_value.to_string(), - ); - } - if let Some(local_var_param_value) = if_none_match { - local_var_req_builder = - local_var_req_builder.header("If-None-Match", local_var_param_value.to_string()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadTableResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadTableResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Load a view from the catalog. The response contains both configuration and view metadata. The configuration, if non-empty is used as additional configuration for the view that overrides catalog configuration. The response also contains the view's full metadata, matching the view metadata JSON file. The catalog configuration may contain credentials that should be used for subsequent requests for the view. The configuration key \"token\" is used to pass an access token to be used as a bearer token for view requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, \"urn:ietf:params:oauth:token-type:jwt=\". - async fn load_view<'namespace, 'view, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/views/{view}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - view = crate::apis::urlencode(view) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadViewResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadViewResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Check if a namespace exists. The response does not contain a body. - async fn namespace_exists<'namespace, 'prefix>( - &self, - namespace: &'namespace str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::HEAD, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Submits a scan for server-side planning. Point-in-time scans are planned by passing snapshot-id to identify the table snapshot to scan. Incremental scans are planned by passing both start-snapshot-id and end-snapshot-id. Requests that include both point in time config properties and incremental config properties are invalid. If the request does not include either incremental or point-in-time config properties, scan planning should produce a point-in-time scan of the latest snapshot in the table's main branch. Responses must include a valid status listed below. A \"cancelled\" status is considered invalid for this endpoint. - When \"completed\" the planning operation has produced plan tasks and file scan tasks that must be returned in the response (not fetched later by calling fetchPlanningResult) - When \"submitted\" the response must include a plan-id used to poll fetchPlanningResult to fetch the planning result when it is ready - When \"failed\" the response must be a valid error response The response for a \"completed\" planning operation includes two types of tasks (file scan tasks and plan tasks) and both may be included in the response. Tasks must not be included for any other response status. Responses that include a plan-id indicate that the service is holding state or performing work for the client. - Clients should use the plan-id to fetch results from fetchPlanningResult when the response status is \"submitted\" - Clients should inform the service if planning results are no longer needed by calling cancelPlanning. Cancellation is not necessary after fetchScanTasks has been used to fetch scan tasks for each plan task. - async fn plan_table_scan<'namespace, 'table, 'prefix, 'plan_table_scan_request>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - plan_table_scan_request: Option, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}/plan", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&plan_table_scan_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::PlanTableScanResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::PlanTableScanResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Register a table using given metadata file location. - async fn register_table<'namespace, 'register_table_request, 'prefix>( - &self, - namespace: &'namespace str, - register_table_request: models::RegisterTableRequest, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/register", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(®ister_table_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadTableResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadTableResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Rename a table from one identifier to another. It's valid to move a table across namespaces, but the server implementation is not required to support it. - async fn rename_table<'rename_table_request, 'prefix>( - &self, - rename_table_request: models::RenameTableRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/tables/rename", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default() - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&rename_table_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Rename a view from one identifier to another. It's valid to move a view across namespaces, but the server implementation is not required to support it. - async fn rename_view<'rename_table_request, 'prefix>( - &self, - rename_table_request: models::RenameTableRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/views/rename", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default() - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&rename_table_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Commit updates to a view. - async fn replace_view<'namespace, 'view, 'commit_view_request, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - commit_view_request: models::CommitViewRequest, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/views/{view}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - view = crate::apis::urlencode(view) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&commit_view_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::LoadViewResult`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::LoadViewResult`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - async fn report_metrics<'namespace, 'table, 'report_metrics_request, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - report_metrics_request: models::ReportMetricsRequest, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}/metrics", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&report_metrics_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Check if a table exists within a given namespace. The response does not contain a body. - async fn table_exists<'namespace, 'table, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::HEAD, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Set and/or remove properties on a namespace. The request body specifies a list of properties to remove and a map of key value pairs to update. Properties that are not in the request are not modified or removed by this call. Server implementations are not required to support namespace properties. - async fn update_properties<'namespace, 'update_namespace_properties_request, 'prefix>( - &self, - namespace: &'namespace str, - update_namespace_properties_request: models::UpdateNamespacePropertiesRequest, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/properties", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&update_namespace_properties_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::UpdateNamespacePropertiesResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::UpdateNamespacePropertiesResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Commit updates to a table. Commits have two parts, requirements and updates. Requirements are assertions that will be validated before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a named ref's snapshot ID has a certain value. Server implementations are required to fail with a 400 status code if any unknown updates or requirements are received. Updates are changes to make to table metadata. For example, after asserting that the current main ref is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new snapshot id. Create table transactions that are started by createTable with `stage-create` set to true are committed using this route. Transactions should include all changes to the table, including table initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is used to ensure that the table was not created concurrently. - async fn update_table<'namespace, 'table, 'commit_table_request, 'prefix>( - &self, - namespace: &'namespace str, - table: &'table str, - commit_table_request: models::CommitTableRequest, - prefix: Option<&'prefix str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - local_var_req_builder = local_var_req_builder.json(&commit_table_request); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::CommitTableResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::CommitTableResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } - - /// Check if a view exists within a given namespace. This request does not return a response body. - async fn view_exists<'namespace, 'view, 'prefix>( - &self, - namespace: &'namespace str, - view: &'view str, - prefix: Option<&'prefix str>, - ) -> Result<(), Error> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!( - "{}/v1{prefix}/namespaces/{namespace}/views/{view}", - local_var_configuration.base_path, - prefix = prefix - .map(|p| format!("/{}", crate::apis::urlencode(p))) - .unwrap_or_default(), - namespace = crate::apis::urlencode(namespace), - view = crate::apis::urlencode(view) - ); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::HEAD, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - Ok(()) - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } -} - -/// struct for typed errors of method [`CatalogApiApi::cancel_planning`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CancelPlanningError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::commit_transaction`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CommitTransactionError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status500(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status502(models::IcebergErrorResponse), - Status504(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::create_namespace`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CreateNamespaceError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status406(models::ErrorModel), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::create_table`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CreateTableError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::create_view`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CreateViewError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::ErrorModel), - Status409(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::drop_namespace`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum DropNamespaceError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::drop_table`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum DropTableError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::drop_view`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum DropViewError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::fetch_planning_result`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum FetchPlanningResultError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::fetch_scan_tasks`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum FetchScanTasksError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::list_namespaces`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ListNamespacesError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::list_tables`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ListTablesError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::list_views`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ListViewsError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::load_credentials`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum LoadCredentialsError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::load_namespace_metadata`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum LoadNamespaceMetadataError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::load_table`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum LoadTableError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::load_view`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum LoadViewError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::namespace_exists`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum NamespaceExistsError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::plan_table_scan`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum PlanTableScanError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status406(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::register_table`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum RegisterTableError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::rename_table`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum RenameTableError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status406(models::ErrorModel), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::rename_view`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum RenameViewError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::ErrorModel), - Status406(models::ErrorModel), - Status409(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::replace_view`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ReplaceViewError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::ErrorModel), - Status409(models::ErrorModel), - Status419(models::IcebergErrorResponse), - Status500(models::ErrorModel), - Status503(models::IcebergErrorResponse), - Status502(models::ErrorModel), - Status504(models::ErrorModel), - Status5XX(models::ErrorModel), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::report_metrics`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ReportMetricsError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::table_exists`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum TableExistsError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::update_properties`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum UpdatePropertiesError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status406(models::ErrorModel), - Status422(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::update_table`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum UpdateTableError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status404(models::IcebergErrorResponse), - Status409(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status500(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status502(models::IcebergErrorResponse), - Status504(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} - -/// struct for typed errors of method [`CatalogApiApi::view_exists`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ViewExistsError { - Status400(), - Status401(), - Status404(), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} diff --git a/crates/sail-catalog-iceberg/src/apis/configuration.rs b/crates/sail-catalog-iceberg/src/apis/configuration.rs deleted file mode 100644 index 23a7f8c68..000000000 --- a/crates/sail-catalog-iceberg/src/apis/configuration.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -#[derive(Debug, Clone)] -pub struct Configuration { - pub base_path: String, - pub user_agent: Option, - pub client: reqwest::Client, - pub basic_auth: Option, - pub oauth_access_token: Option, - pub bearer_access_token: Option, - pub api_key: Option, -} - -pub type BasicAuth = (String, Option); - -#[derive(Debug, Clone)] -pub struct ApiKey { - pub prefix: Option, - pub key: String, -} - -impl Configuration { - pub fn new() -> Configuration { - Configuration::default() - } -} - -impl Default for Configuration { - fn default() -> Self { - Configuration { - base_path: "https://localhost".to_owned(), - user_agent: Some("Sail".to_owned()), - client: reqwest::Client::new(), - basic_auth: None, - oauth_access_token: None, - bearer_access_token: None, - api_key: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/apis/configuration_api_api.rs b/crates/sail-catalog-iceberg/src/apis/configuration_api_api.rs deleted file mode 100644 index ad49540cc..000000000 --- a/crates/sail-catalog-iceberg/src/apis/configuration_api_api.rs +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use std::sync::Arc; - -use async_trait::async_trait; -use reqwest; -use serde::de::Error as _; -use serde::{Deserialize, Serialize}; - -use super::{configuration, Error}; -use crate::apis::{ContentType, ResponseContent}; -use crate::models; - -#[async_trait] -pub trait ConfigurationApiApi: Send + Sync { - /// GET /v1/config - /// - /// All REST clients should first call this route to get catalog configuration properties from the server to configure the catalog and its HTTP client. Configuration from the server consists of two sets of key/value pairs. - defaults - properties that should be used as default configuration; applied before client configuration - overrides - properties that should be used to override client configuration; applied after defaults and client configuration Catalog configuration is constructed by setting the defaults, then client- provided configuration, and finally overrides. The final property set is then used to configure the catalog. For example, a default configuration property might set the size of the client pool, which can be replaced with a client-specific setting. An override might be used to set the warehouse location, which is stored on the server rather than in client configuration. Common catalog configuration settings are documented at https://iceberg.apache.org/docs/latest/configuration/#catalog-properties The catalog configuration also holds an optional `endpoints` field that contains information about the endpoints supported by the server. If a server does not send the `endpoints` field, a default set of endpoints is assumed: - GET /v1/{prefix}/namespaces - POST /v1/{prefix}/namespaces - GET /v1/{prefix}/namespaces/{namespace} - DELETE /v1/{prefix}/namespaces/{namespace} - POST /v1/{prefix}/namespaces/{namespace}/properties - GET /v1/{prefix}/namespaces/{namespace}/tables - POST /v1/{prefix}/namespaces/{namespace}/tables - GET /v1/{prefix}/namespaces/{namespace}/tables/{table} - POST /v1/{prefix}/namespaces/{namespace}/tables/{table} - DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table} - POST /v1/{prefix}/namespaces/{namespace}/register - POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics - POST /v1/{prefix}/tables/rename - POST /v1/{prefix}/transactions/commit - async fn get_config<'warehouse>( - &self, - warehouse: Option<&'warehouse str>, - ) -> Result>; -} - -pub struct ConfigurationApiApiClient { - configuration: Arc, -} - -impl ConfigurationApiApiClient { - pub fn new(configuration: Arc) -> Self { - Self { configuration } - } -} - -#[async_trait] -impl ConfigurationApiApi for ConfigurationApiApiClient { - /// All REST clients should first call this route to get catalog configuration properties from the server to configure the catalog and its HTTP client. Configuration from the server consists of two sets of key/value pairs. - defaults - properties that should be used as default configuration; applied before client configuration - overrides - properties that should be used to override client configuration; applied after defaults and client configuration Catalog configuration is constructed by setting the defaults, then client- provided configuration, and finally overrides. The final property set is then used to configure the catalog. For example, a default configuration property might set the size of the client pool, which can be replaced with a client-specific setting. An override might be used to set the warehouse location, which is stored on the server rather than in client configuration. Common catalog configuration settings are documented at https://iceberg.apache.org/docs/latest/configuration/#catalog-properties The catalog configuration also holds an optional `endpoints` field that contains information about the endpoints supported by the server. If a server does not send the `endpoints` field, a default set of endpoints is assumed: - GET /v1/{prefix}/namespaces - POST /v1/{prefix}/namespaces - GET /v1/{prefix}/namespaces/{namespace} - DELETE /v1/{prefix}/namespaces/{namespace} - POST /v1/{prefix}/namespaces/{namespace}/properties - GET /v1/{prefix}/namespaces/{namespace}/tables - POST /v1/{prefix}/namespaces/{namespace}/tables - GET /v1/{prefix}/namespaces/{namespace}/tables/{table} - POST /v1/{prefix}/namespaces/{namespace}/tables/{table} - DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table} - POST /v1/{prefix}/namespaces/{namespace}/register - POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics - POST /v1/{prefix}/tables/rename - POST /v1/{prefix}/transactions/commit - async fn get_config<'warehouse>( - &self, - warehouse: Option<&'warehouse str>, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!("{}/v1/config", local_var_configuration.base_path); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::GET, local_var_uri_str.as_str()); - - if let Some(ref param_value) = warehouse { - local_var_req_builder = - local_var_req_builder.query(&[("warehouse", ¶m_value.to_string())]); - } - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::CatalogConfig`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::CatalogConfig`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } -} - -/// struct for typed errors of method [`ConfigurationApiApi::get_config`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum GetConfigError { - Status400(models::IcebergErrorResponse), - Status401(models::IcebergErrorResponse), - Status403(models::IcebergErrorResponse), - Status419(models::IcebergErrorResponse), - Status503(models::IcebergErrorResponse), - Status5XX(models::IcebergErrorResponse), - UnknownValue(serde_json::Value), -} diff --git a/crates/sail-catalog-iceberg/src/apis/mod.rs b/crates/sail-catalog-iceberg/src/apis/mod.rs deleted file mode 100644 index ae3a505cc..000000000 --- a/crates/sail-catalog-iceberg/src/apis/mod.rs +++ /dev/null @@ -1,172 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{error, fmt}; - -#[derive(Debug, Clone)] -pub struct ResponseContent { - pub status: reqwest::StatusCode, - pub content: String, - pub entity: Option, -} - -#[derive(Debug)] -pub enum Error { - Reqwest(reqwest::Error), - Serde(serde_json::Error), - Io(std::io::Error), - ResponseError(ResponseContent), -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (module, e) = match self { - Error::Reqwest(e) => ("reqwest", e.to_string()), - Error::Serde(e) => ("serde", e.to_string()), - Error::Io(e) => ("IO", e.to_string()), - Error::ResponseError(e) => ("response", format!("status code {}", e.status)), - }; - write!(f, "error in {}: {}", module, e) - } -} - -impl error::Error for Error { - fn source(&self) -> Option<&(dyn error::Error + 'static)> { - Some(match self { - Error::Reqwest(e) => e, - Error::Serde(e) => e, - Error::Io(e) => e, - Error::ResponseError(_) => return None, - }) - } -} - -impl From for Error { - fn from(e: reqwest::Error) -> Self { - Error::Reqwest(e) - } -} - -impl From for Error { - fn from(e: serde_json::Error) -> Self { - Error::Serde(e) - } -} - -impl From for Error { - fn from(e: std::io::Error) -> Self { - Error::Io(e) - } -} - -pub fn urlencode>(s: T) -> String { - ::url::form_urlencoded::byte_serialize(s.as_ref().as_bytes()).collect() -} - -pub fn parse_deep_object(prefix: &str, value: &serde_json::Value) -> Vec<(String, String)> { - if let serde_json::Value::Object(object) = value { - let mut params = vec![]; - - for (key, value) in object { - match value { - serde_json::Value::Object(_) => params.append(&mut parse_deep_object( - &format!("{}[{}]", prefix, key), - value, - )), - serde_json::Value::Array(array) => { - for (i, value) in array.iter().enumerate() { - params.append(&mut parse_deep_object( - &format!("{}[{}][{}]", prefix, key, i), - value, - )); - } - } - serde_json::Value::String(s) => { - params.push((format!("{}[{}]", prefix, key), s.clone())) - } - _ => params.push((format!("{}[{}]", prefix, key), value.to_string())), - } - } - - return params; - } - - unimplemented!("Only objects are supported with style=deepObject") -} - -/// Internal use only -/// A content type supported by this client. -enum ContentType { - Json, - Text, - Unsupported(String), -} - -impl From<&str> for ContentType { - fn from(content_type: &str) -> Self { - if content_type.starts_with("application") && content_type.contains("json") { - return Self::Json; - } else if content_type.starts_with("text/plain") { - return Self::Text; - } else { - return Self::Unsupported(content_type.to_string()); - } - } -} - -pub mod catalog_api_api; -pub mod configuration_api_api; -pub mod o_auth2_api_api; - -pub mod configuration; - -use std::sync::Arc; - -pub trait Api { - fn catalog_api_api(&self) -> &dyn catalog_api_api::CatalogApiApi; - fn configuration_api_api(&self) -> &dyn configuration_api_api::ConfigurationApiApi; - fn o_auth2_api_api(&self) -> &dyn o_auth2_api_api::OAuth2ApiApi; -} - -pub struct ApiClient { - catalog_api_api: Box, - configuration_api_api: Box, - o_auth2_api_api: Box, -} - -impl ApiClient { - pub fn new(configuration: Arc) -> Self { - Self { - catalog_api_api: Box::new(catalog_api_api::CatalogApiApiClient::new( - configuration.clone(), - )), - configuration_api_api: Box::new(configuration_api_api::ConfigurationApiApiClient::new( - configuration.clone(), - )), - o_auth2_api_api: Box::new(o_auth2_api_api::OAuth2ApiApiClient::new( - configuration.clone(), - )), - } - } -} - -impl Api for ApiClient { - fn catalog_api_api(&self) -> &dyn catalog_api_api::CatalogApiApi { - self.catalog_api_api.as_ref() - } - fn configuration_api_api(&self) -> &dyn configuration_api_api::ConfigurationApiApi { - self.configuration_api_api.as_ref() - } - fn o_auth2_api_api(&self) -> &dyn o_auth2_api_api::OAuth2ApiApi { - self.o_auth2_api_api.as_ref() - } -} diff --git a/crates/sail-catalog-iceberg/src/apis/o_auth2_api_api.rs b/crates/sail-catalog-iceberg/src/apis/o_auth2_api_api.rs deleted file mode 100644 index f30f84105..000000000 --- a/crates/sail-catalog-iceberg/src/apis/o_auth2_api_api.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use std::sync::Arc; - -use async_trait::async_trait; -use reqwest; -use serde::de::Error as _; -use serde::{Deserialize, Serialize}; - -use super::{configuration, Error}; -use crate::apis::{ContentType, ResponseContent}; -use crate::models; - -#[async_trait] -pub trait OAuth2ApiApi: Send + Sync { - /// POST /v1/oauth/tokens - /// - /// The `oauth/tokens` endpoint is **DEPRECATED for REMOVAL**. It is _not_ recommended to implement this endpoint, unless you are fully aware of the potential security implications. All clients are encouraged to explicitly set the configuration property `oauth2-server-uri` to the correct OAuth endpoint. Deprecated since Iceberg (Java) 1.6.0. The endpoint and related types will be removed from this spec in Iceberg (Java) 2.0. See [Security improvements in the Iceberg REST specification](https://github.com/apache/iceberg/issues/10537) Exchange credentials for a token using the OAuth2 client credentials flow or token exchange. This endpoint is used for three purposes - 1. To exchange client credentials (client ID and secret) for an access token This uses the client credentials flow. 2. To exchange a client token and an identity token for a more specific access token This uses the token exchange flow. 3. To exchange an access token for one with the same claims and a refreshed expiration period This uses the token exchange flow. For example, a catalog client may be configured with client credentials from the OAuth2 Authorization flow. This client would exchange its client ID and secret for an access token using the client credentials request with this endpoint (1). Subsequent requests would then use that access token. Some clients may also handle sessions that have additional user context. These clients would use the token exchange flow to exchange a user token (the \"subject\" token) from the session for a more specific access token for that user, using the catalog's access token as the \"actor\" token (2). The user ID token is the \"subject\" token and can be any token type allowed by the OAuth2 token exchange flow, including a unsecured JWT token with a sub claim. This request should use the catalog's bearer token in the \"Authorization\" header. Clients may also use the token exchange flow to refresh a token that is about to expire by sending a token exchange request (3). The request's \"subject\" token should be the expiring token. This request should use the subject token in the \"Authorization\" header. - async fn get_token< - 'grant_type, - 'scope, - 'client_id, - 'client_secret, - 'requested_token_type, - 'subject_token, - 'subject_token_type, - 'actor_token, - 'actor_token_type, - >( - &self, - grant_type: Option<&'grant_type str>, - scope: Option<&'scope str>, - client_id: Option<&'client_id str>, - client_secret: Option<&'client_secret str>, - requested_token_type: Option, - subject_token: Option<&'subject_token str>, - subject_token_type: Option, - actor_token: Option<&'actor_token str>, - actor_token_type: Option, - ) -> Result>; -} - -pub struct OAuth2ApiApiClient { - configuration: Arc, -} - -impl OAuth2ApiApiClient { - pub fn new(configuration: Arc) -> Self { - Self { configuration } - } -} - -#[async_trait] -impl OAuth2ApiApi for OAuth2ApiApiClient { - /// The `oauth/tokens` endpoint is **DEPRECATED for REMOVAL**. It is _not_ recommended to implement this endpoint, unless you are fully aware of the potential security implications. All clients are encouraged to explicitly set the configuration property `oauth2-server-uri` to the correct OAuth endpoint. Deprecated since Iceberg (Java) 1.6.0. The endpoint and related types will be removed from this spec in Iceberg (Java) 2.0. See [Security improvements in the Iceberg REST specification](https://github.com/apache/iceberg/issues/10537) Exchange credentials for a token using the OAuth2 client credentials flow or token exchange. This endpoint is used for three purposes - 1. To exchange client credentials (client ID and secret) for an access token This uses the client credentials flow. 2. To exchange a client token and an identity token for a more specific access token This uses the token exchange flow. 3. To exchange an access token for one with the same claims and a refreshed expiration period This uses the token exchange flow. For example, a catalog client may be configured with client credentials from the OAuth2 Authorization flow. This client would exchange its client ID and secret for an access token using the client credentials request with this endpoint (1). Subsequent requests would then use that access token. Some clients may also handle sessions that have additional user context. These clients would use the token exchange flow to exchange a user token (the \"subject\" token) from the session for a more specific access token for that user, using the catalog's access token as the \"actor\" token (2). The user ID token is the \"subject\" token and can be any token type allowed by the OAuth2 token exchange flow, including a unsecured JWT token with a sub claim. This request should use the catalog's bearer token in the \"Authorization\" header. Clients may also use the token exchange flow to refresh a token that is about to expire by sending a token exchange request (3). The request's \"subject\" token should be the expiring token. This request should use the subject token in the \"Authorization\" header. - async fn get_token< - 'grant_type, - 'scope, - 'client_id, - 'client_secret, - 'requested_token_type, - 'subject_token, - 'subject_token_type, - 'actor_token, - 'actor_token_type, - >( - &self, - grant_type: Option<&'grant_type str>, - scope: Option<&'scope str>, - client_id: Option<&'client_id str>, - client_secret: Option<&'client_secret str>, - requested_token_type: Option, - subject_token: Option<&'subject_token str>, - subject_token_type: Option, - actor_token: Option<&'actor_token str>, - actor_token_type: Option, - ) -> Result> { - let local_var_configuration = &self.configuration; - - let local_var_client = &local_var_configuration.client; - - let local_var_uri_str = format!("{}/v1/oauth/tokens", local_var_configuration.base_path); - let mut local_var_req_builder = - local_var_client.request(reqwest::Method::POST, local_var_uri_str.as_str()); - - if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { - local_var_req_builder = local_var_req_builder - .header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); - } - if let Some(ref local_var_token) = local_var_configuration.oauth_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - if let Some(ref local_var_token) = local_var_configuration.bearer_access_token { - local_var_req_builder = local_var_req_builder.bearer_auth(local_var_token.to_owned()); - }; - let mut local_var_form_params = std::collections::HashMap::new(); - if let Some(local_var_param_value) = grant_type { - local_var_form_params.insert("grant_type", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = scope { - local_var_form_params.insert("scope", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = client_id { - local_var_form_params.insert("client_id", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = client_secret { - local_var_form_params.insert("client_secret", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = requested_token_type { - local_var_form_params.insert("requested_token_type", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = subject_token { - local_var_form_params.insert("subject_token", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = subject_token_type { - local_var_form_params.insert("subject_token_type", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = actor_token { - local_var_form_params.insert("actor_token", local_var_param_value.to_string()); - } - if let Some(local_var_param_value) = actor_token_type { - local_var_form_params.insert("actor_token_type", local_var_param_value.to_string()); - } - local_var_req_builder = local_var_req_builder.form(&local_var_form_params); - - let local_var_req = local_var_req_builder.build()?; - let local_var_resp = local_var_client.execute(local_var_req).await?; - - let local_var_status = local_var_resp.status(); - let local_var_content_type = local_var_resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("application/octet-stream"); - let local_var_content_type = super::ContentType::from(local_var_content_type); - let local_var_content = local_var_resp.text().await?; - - if !local_var_status.is_client_error() && !local_var_status.is_server_error() { - match local_var_content_type { - ContentType::Json => serde_json::from_str(&local_var_content).map_err(Error::from), - ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::OAuthTokenResponse`"))), - ContentType::Unsupported(local_var_unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{local_var_unknown_type}` content type response that cannot be converted to `models::OAuthTokenResponse`")))), - } - } else { - let local_var_entity: Option = - serde_json::from_str(&local_var_content).ok(); - let local_var_error = ResponseContent { - status: local_var_status, - content: local_var_content, - entity: local_var_entity, - }; - Err(Error::ResponseError(local_var_error)) - } - } -} - -/// struct for typed errors of method [`OAuth2ApiApi::get_token`] -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum GetTokenError { - Status400(models::OAuthError), - Status401(models::OAuthError), - Status5XX(models::OAuthError), - UnknownValue(serde_json::Value), -} diff --git a/crates/sail-catalog-iceberg/src/lib.rs b/crates/sail-catalog-iceberg/src/lib.rs index 740411d48..816b6718b 100644 --- a/crates/sail-catalog-iceberg/src/lib.rs +++ b/crates/sail-catalog-iceberg/src/lib.rs @@ -10,77 +10,90 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![expect(unused_imports)] -#![expect(dead_code)] -#![expect(clippy::all)] - -pub mod apis; -mod models; +mod adapter; mod provider; -pub use provider::{ - IcebergRestCatalogProvider, REST_CATALOG_PROP_PREFIX, REST_CATALOG_PROP_URI, - REST_CATALOG_PROP_WAREHOUSE, -}; - -#[cfg(test)] -mod table_update_tests { - //! Regression tests for the discriminated `TableUpdate` / `ViewUpdate` - //! models. Before the spec was given a discriminator, these were generated - //! as flat structs with every field required, so any real update (e.g. - //! `set-properties`) failed to deserialize with `missing field uuid`. - #![expect(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - use crate::models; +#[expect(unused_imports)] +#[expect(clippy::enum_variant_names)] +#[expect(clippy::match_overlapping_arm)] +#[expect(clippy::doc_lazy_continuation)] +pub mod r#gen { + include!(concat!(env!("OUT_DIR"), "/iceberg_rest_catalog_gen.rs")); - #[test] - fn commit_table_request_accepts_set_properties_update() { - let request: models::CommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [{"type": "assert-table-uuid", "uuid": "11111111-1111-1111-1111-111111111111"}], - "updates": [{"action": "set-properties", "updates": {"k": "v"}}] - })) - .expect("set-properties commit must deserialize"); - assert_eq!(request.updates.len(), 1); - assert!(matches!( - request.updates[0], - models::TableUpdate::SetPropertiesUpdate {} - )); + // The discriminator is missing for `ReportMetricsRequest` in the OpenAPI specification, + // so this schema is excluded from generation and defined manually. + // See also: https://github.com/apache/iceberg/issues/12696 + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] + #[serde(tag = "report-type")] + pub enum ReportMetricsRequest { + #[serde(rename = "scan-report")] + ScanReport(Box), + #[serde(rename = "commit-report")] + CommitReport(Box), } - #[test] - fn table_update_discriminates_on_action() { - let cases = [ - ("assign-uuid", "AssignUuidUpdate"), - ("add-snapshot", "AddSnapshotUpdate"), - ("set-location", "SetLocationUpdate"), - ("remove-properties", "RemovePropertiesUpdate"), - ("upgrade-format-version", "UpgradeFormatVersionUpdate"), - ]; - for (action, _name) in cases { - let update: models::TableUpdate = - serde_json::from_value(serde_json::json!({"action": action})) - .unwrap_or_else(|err| panic!("action {action} must deserialize: {err}")); - // Round-trips back to the same tagged action. - let value = serde_json::to_value(&update).unwrap(); - assert_eq!(value["action"], serde_json::json!(action)); + // TODO: Generate this schema once the OpenAPI generator supports objects with both fixed + // properties and additional properties. + pub mod snapshot { + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] + pub enum Operation { + #[serde(rename = "append")] + Append, + #[serde(rename = "replace")] + Replace, + #[serde(rename = "overwrite")] + Overwrite, + #[serde(rename = "delete")] + Delete, } - } - #[test] - fn view_update_accepts_set_properties() { - let request: models::CommitViewRequest = serde_json::from_value(serde_json::json!({ - "updates": [{"action": "set-properties", "updates": {"k": "v"}}] - })) - .expect("view set-properties commit must deserialize"); - assert!(matches!( - request.updates[0], - models::ViewUpdate::SetPropertiesUpdate {} - )); + impl std::fmt::Display for Operation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Append => f.write_str("append"), + Self::Replace => f.write_str("replace"), + Self::Overwrite => f.write_str("overwrite"), + Self::Delete => f.write_str("delete"), + } + } + } + + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] + pub struct Summary { + pub operation: Operation, + #[serde(flatten)] + pub additional_properties: std::collections::HashMap, + } } - #[test] - fn unknown_action_is_rejected() { - let result: Result = - serde_json::from_value(serde_json::json!({"action": "not-a-real-update"})); - assert!(result.is_err(), "unknown action must not deserialize"); + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] + pub struct Snapshot { + #[serde(rename = "added-rows")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub added_rows: Option, + #[serde(rename = "first-row-id")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first_row_id: Option, + #[serde(rename = "manifest-list")] + pub manifest_list: String, + #[serde(rename = "parent-snapshot-id")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_snapshot_id: Option, + #[serde(rename = "schema-id")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema_id: Option, + #[serde(rename = "sequence-number")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sequence_number: Option, + #[serde(rename = "snapshot-id")] + pub snapshot_id: i64, + pub summary: snapshot::Summary, + #[serde(rename = "timestamp-ms")] + pub timestamp_ms: i64, } } + +pub use provider::{ + IcebergRestCatalogOptions, IcebergRestCatalogProvider, REST_CATALOG_PROP_PREFIX, + REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, +}; diff --git a/crates/sail-catalog-iceberg/src/models/add_encryption_key_update.rs b/crates/sail-catalog-iceberg/src/models/add_encryption_key_update.rs deleted file mode 100644 index 8809a8505..000000000 --- a/crates/sail-catalog-iceberg/src/models/add_encryption_key_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AddEncryptionKeyUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "encryption-key")] - pub encryption_key: Box, -} - -impl AddEncryptionKeyUpdate { - pub fn new(encryption_key: models::EncryptedKey) -> AddEncryptionKeyUpdate { - AddEncryptionKeyUpdate { - action: None, - encryption_key: Box::new(encryption_key), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/add_partition_spec_update.rs b/crates/sail-catalog-iceberg/src/models/add_partition_spec_update.rs deleted file mode 100644 index 32a54b26c..000000000 --- a/crates/sail-catalog-iceberg/src/models/add_partition_spec_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AddPartitionSpecUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "spec")] - pub spec: Box, -} - -impl AddPartitionSpecUpdate { - pub fn new(spec: models::PartitionSpec) -> AddPartitionSpecUpdate { - AddPartitionSpecUpdate { - action: None, - spec: Box::new(spec), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/add_schema_update.rs b/crates/sail-catalog-iceberg/src/models/add_schema_update.rs deleted file mode 100644 index fc0ea9665..000000000 --- a/crates/sail-catalog-iceberg/src/models/add_schema_update.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AddSchemaUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "schema")] - pub schema: Box, - /// This optional field is **DEPRECATED for REMOVAL** since it more safe to handle this internally, and shouldn't be exposed to the clients. The highest assigned column ID for the table. This is used to ensure columns are always assigned an unused ID when evolving schemas. When omitted, it will be computed on the server side. - #[serde(rename = "last-column-id", skip_serializing_if = "Option::is_none")] - pub last_column_id: Option, -} - -impl AddSchemaUpdate { - pub fn new(schema: models::Schema) -> AddSchemaUpdate { - AddSchemaUpdate { - action: None, - schema: Box::new(schema), - last_column_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/add_snapshot_update.rs b/crates/sail-catalog-iceberg/src/models/add_snapshot_update.rs deleted file mode 100644 index 8f91687f3..000000000 --- a/crates/sail-catalog-iceberg/src/models/add_snapshot_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AddSnapshotUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "snapshot")] - pub snapshot: Box, -} - -impl AddSnapshotUpdate { - pub fn new(snapshot: models::Snapshot) -> AddSnapshotUpdate { - AddSnapshotUpdate { - action: None, - snapshot: Box::new(snapshot), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/add_sort_order_update.rs b/crates/sail-catalog-iceberg/src/models/add_sort_order_update.rs deleted file mode 100644 index 35e0ba570..000000000 --- a/crates/sail-catalog-iceberg/src/models/add_sort_order_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AddSortOrderUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "sort-order")] - pub sort_order: Box, -} - -impl AddSortOrderUpdate { - pub fn new(sort_order: models::SortOrder) -> AddSortOrderUpdate { - AddSortOrderUpdate { - action: None, - sort_order: Box::new(sort_order), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/add_view_version_update.rs b/crates/sail-catalog-iceberg/src/models/add_view_version_update.rs deleted file mode 100644 index 529d813a0..000000000 --- a/crates/sail-catalog-iceberg/src/models/add_view_version_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AddViewVersionUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "view-version")] - pub view_version: Box, -} - -impl AddViewVersionUpdate { - pub fn new(view_version: models::ViewVersion) -> AddViewVersionUpdate { - AddViewVersionUpdate { - action: None, - view_version: Box::new(view_version), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/and_or_expression.rs b/crates/sail-catalog-iceberg/src/models/and_or_expression.rs deleted file mode 100644 index a0d27c338..000000000 --- a/crates/sail-catalog-iceberg/src/models/and_or_expression.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AndOrExpression { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "left")] - pub left: Box, - #[serde(rename = "right")] - pub right: Box, -} - -impl AndOrExpression { - pub fn new( - r#type: String, - left: models::Expression, - right: models::Expression, - ) -> AndOrExpression { - AndOrExpression { - r#type, - left: Box::new(left), - right: Box::new(right), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_create.rs b/crates/sail-catalog-iceberg/src/models/assert_create.rs deleted file mode 100644 index 924fc6528..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_create.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertCreate : The table must not already exist; used for create transactions -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertCreate { - #[serde(rename = "type")] - pub r#type: String, -} - -impl AssertCreate { - /// The table must not already exist; used for create transactions - pub fn new(r#type: String) -> AssertCreate { - AssertCreate { r#type } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_current_schema_id.rs b/crates/sail-catalog-iceberg/src/models/assert_current_schema_id.rs deleted file mode 100644 index ea87fddc6..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_current_schema_id.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertCurrentSchemaId : The table's current schema id must match the requirement's `current-schema-id` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertCurrentSchemaId { - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub r#type: Option, - #[serde(rename = "current-schema-id")] - pub current_schema_id: i32, -} - -impl AssertCurrentSchemaId { - /// The table's current schema id must match the requirement's `current-schema-id` - pub fn new(current_schema_id: i32) -> AssertCurrentSchemaId { - AssertCurrentSchemaId { - r#type: None, - current_schema_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_default_sort_order_id.rs b/crates/sail-catalog-iceberg/src/models/assert_default_sort_order_id.rs deleted file mode 100644 index 4d04b5ed6..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_default_sort_order_id.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertDefaultSortOrderId : The table's default sort order id must match the requirement's `default-sort-order-id` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertDefaultSortOrderId { - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub r#type: Option, - #[serde(rename = "default-sort-order-id")] - pub default_sort_order_id: i32, -} - -impl AssertDefaultSortOrderId { - /// The table's default sort order id must match the requirement's `default-sort-order-id` - pub fn new(default_sort_order_id: i32) -> AssertDefaultSortOrderId { - AssertDefaultSortOrderId { - r#type: None, - default_sort_order_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_default_spec_id.rs b/crates/sail-catalog-iceberg/src/models/assert_default_spec_id.rs deleted file mode 100644 index 22b76e67a..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_default_spec_id.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertDefaultSpecId : The table's default spec id must match the requirement's `default-spec-id` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertDefaultSpecId { - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub r#type: Option, - #[serde(rename = "default-spec-id")] - pub default_spec_id: i32, -} - -impl AssertDefaultSpecId { - /// The table's default spec id must match the requirement's `default-spec-id` - pub fn new(default_spec_id: i32) -> AssertDefaultSpecId { - AssertDefaultSpecId { - r#type: None, - default_spec_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_last_assigned_field_id.rs b/crates/sail-catalog-iceberg/src/models/assert_last_assigned_field_id.rs deleted file mode 100644 index ebcef845d..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_last_assigned_field_id.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertLastAssignedFieldId : The table's last assigned column id must match the requirement's `last-assigned-field-id` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertLastAssignedFieldId { - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub r#type: Option, - #[serde(rename = "last-assigned-field-id")] - pub last_assigned_field_id: i32, -} - -impl AssertLastAssignedFieldId { - /// The table's last assigned column id must match the requirement's `last-assigned-field-id` - pub fn new(last_assigned_field_id: i32) -> AssertLastAssignedFieldId { - AssertLastAssignedFieldId { - r#type: None, - last_assigned_field_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_last_assigned_partition_id.rs b/crates/sail-catalog-iceberg/src/models/assert_last_assigned_partition_id.rs deleted file mode 100644 index 85b78b535..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_last_assigned_partition_id.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertLastAssignedPartitionId : The table's last assigned partition id must match the requirement's `last-assigned-partition-id` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertLastAssignedPartitionId { - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub r#type: Option, - #[serde(rename = "last-assigned-partition-id")] - pub last_assigned_partition_id: i32, -} - -impl AssertLastAssignedPartitionId { - /// The table's last assigned partition id must match the requirement's `last-assigned-partition-id` - pub fn new(last_assigned_partition_id: i32) -> AssertLastAssignedPartitionId { - AssertLastAssignedPartitionId { - r#type: None, - last_assigned_partition_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs b/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs deleted file mode 100644 index 2c9e2c562..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_ref_snapshot_id.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertRefSnapshotId : The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`. The `snapshot-id` field is required in this object, but in the case of a `null` the ref must not already exist. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertRefSnapshotId { - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub r#type: Option, - #[serde(rename = "ref")] - pub r#ref: String, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, -} - -impl AssertRefSnapshotId { - /// The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`. The `snapshot-id` field is required in this object, but in the case of a `null` the ref must not already exist. - pub fn new(r#ref: String, snapshot_id: i64) -> AssertRefSnapshotId { - AssertRefSnapshotId { - r#type: None, - r#ref, - snapshot_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_table_uuid.rs b/crates/sail-catalog-iceberg/src/models/assert_table_uuid.rs deleted file mode 100644 index 80779f45e..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_table_uuid.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertTableUuid : The table UUID must match the requirement's `uuid` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertTableUuid { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "uuid")] - pub uuid: String, -} - -impl AssertTableUuid { - /// The table UUID must match the requirement's `uuid` - pub fn new(r#type: String, uuid: String) -> AssertTableUuid { - AssertTableUuid { r#type, uuid } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assert_view_uuid.rs b/crates/sail-catalog-iceberg/src/models/assert_view_uuid.rs deleted file mode 100644 index 82301ded5..000000000 --- a/crates/sail-catalog-iceberg/src/models/assert_view_uuid.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssertViewUuid : The view UUID must match the requirement's `uuid` -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssertViewUuid { - #[serde(rename = "type")] - pub r#type: Type, - #[serde(rename = "uuid")] - pub uuid: String, -} - -impl AssertViewUuid { - /// The view UUID must match the requirement's `uuid` - pub fn new(r#type: Type, uuid: String) -> AssertViewUuid { - AssertViewUuid { r#type, uuid } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Type { - #[serde(rename = "assert-view-uuid")] - AssertViewUuid, -} - -impl Default for Type { - fn default() -> Type { - Self::AssertViewUuid - } -} diff --git a/crates/sail-catalog-iceberg/src/models/assign_uuid_update.rs b/crates/sail-catalog-iceberg/src/models/assign_uuid_update.rs deleted file mode 100644 index 370440946..000000000 --- a/crates/sail-catalog-iceberg/src/models/assign_uuid_update.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// AssignUuidUpdate : Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AssignUuidUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "uuid")] - pub uuid: String, -} - -impl AssignUuidUpdate { - /// Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned - pub fn new(uuid: String) -> AssignUuidUpdate { - AssignUuidUpdate { action: None, uuid } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/async_planning_result.rs b/crates/sail-catalog-iceberg/src/models/async_planning_result.rs deleted file mode 100644 index d40aee297..000000000 --- a/crates/sail-catalog-iceberg/src/models/async_planning_result.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct AsyncPlanningResult { - #[serde(rename = "status")] - pub status: models::PlanStatus, - /// ID used to track a planning request - #[serde(rename = "plan-id", skip_serializing_if = "Option::is_none")] - pub plan_id: Option, -} - -impl AsyncPlanningResult { - pub fn new(status: models::PlanStatus) -> AsyncPlanningResult { - AsyncPlanningResult { - status, - plan_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/base_update.rs b/crates/sail-catalog-iceberg/src/models/base_update.rs deleted file mode 100644 index 634b41995..000000000 --- a/crates/sail-catalog-iceberg/src/models/base_update.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "action")] -pub enum BaseUpdate { - #[serde(rename = "add-encryption-key")] - AddEncryptionKeyUpdate {}, - #[serde(rename = "add-schema")] - AddSchemaUpdate {}, - #[serde(rename = "add-snapshot")] - AddSnapshotUpdate {}, - #[serde(rename = "add-sort-order")] - AddSortOrderUpdate {}, - #[serde(rename = "add-spec")] - AddPartitionSpecUpdate {}, - #[serde(rename = "add-view-version")] - AddViewVersionUpdate {}, - #[serde(rename = "assign-uuid")] - AssignUuidUpdate {}, - #[serde(rename = "remove-encryption-key")] - RemoveEncryptionKeyUpdate {}, - #[serde(rename = "remove-partition-specs")] - RemovePartitionSpecsUpdate {}, - #[serde(rename = "remove-partition-statistics")] - RemovePartitionStatisticsUpdate {}, - #[serde(rename = "remove-properties")] - RemovePropertiesUpdate {}, - #[serde(rename = "remove-schemas")] - RemoveSchemasUpdate {}, - #[serde(rename = "remove-snapshot-ref")] - RemoveSnapshotRefUpdate {}, - #[serde(rename = "remove-snapshots")] - RemoveSnapshotsUpdate {}, - #[serde(rename = "remove-statistics")] - RemoveStatisticsUpdate {}, - #[serde(rename = "set-current-schema")] - SetCurrentSchemaUpdate {}, - #[serde(rename = "set-current-view-version")] - SetCurrentViewVersionUpdate {}, - #[serde(rename = "set-default-sort-order")] - SetDefaultSortOrderUpdate {}, - #[serde(rename = "set-default-spec")] - SetDefaultSpecUpdate {}, - #[serde(rename = "set-location")] - SetLocationUpdate {}, - #[serde(rename = "set-partition-statistics")] - SetPartitionStatisticsUpdate {}, - #[serde(rename = "set-properties")] - SetPropertiesUpdate {}, - #[serde(rename = "set-snapshot-ref")] - SetSnapshotRefUpdate {}, - #[serde(rename = "set-statistics")] - SetStatisticsUpdate {}, - #[serde(rename = "upgrade-format-version")] - UpgradeFormatVersionUpdate {}, -} - -impl Default for BaseUpdate { - fn default() -> Self { - Self::AddEncryptionKeyUpdate {} - } -} diff --git a/crates/sail-catalog-iceberg/src/models/blob_metadata.rs b/crates/sail-catalog-iceberg/src/models/blob_metadata.rs deleted file mode 100644 index 004c5eef3..000000000 --- a/crates/sail-catalog-iceberg/src/models/blob_metadata.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct BlobMetadata { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "sequence-number")] - pub sequence_number: i64, - #[serde(rename = "fields")] - pub fields: Vec, - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl BlobMetadata { - pub fn new( - r#type: String, - snapshot_id: i64, - sequence_number: i64, - fields: Vec, - ) -> BlobMetadata { - BlobMetadata { - r#type, - snapshot_id, - sequence_number, - fields, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/catalog_config.rs b/crates/sail-catalog-iceberg/src/models/catalog_config.rs deleted file mode 100644 index c49e1f859..000000000 --- a/crates/sail-catalog-iceberg/src/models/catalog_config.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// CatalogConfig : Server-provided configuration for the catalog. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CatalogConfig { - /// Properties that should be used to override client configuration; applied after defaults and client configuration. - #[serde(rename = "overrides")] - pub overrides: std::collections::HashMap, - /// Properties that should be used as default configuration; applied before client configuration. - #[serde(rename = "defaults")] - pub defaults: std::collections::HashMap, - /// A list of endpoints that the server supports. The format of each endpoint must be \" \". The HTTP verb and the resource path must be separated by a space character. - #[serde(rename = "endpoints", skip_serializing_if = "Option::is_none")] - pub endpoints: Option>, -} - -impl CatalogConfig { - /// Server-provided configuration for the catalog. - pub fn new( - overrides: std::collections::HashMap, - defaults: std::collections::HashMap, - ) -> CatalogConfig { - CatalogConfig { - overrides, - defaults, - endpoints: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/commit_report.rs b/crates/sail-catalog-iceberg/src/models/commit_report.rs deleted file mode 100644 index 992a99d51..000000000 --- a/crates/sail-catalog-iceberg/src/models/commit_report.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CommitReport { - #[serde(rename = "table-name")] - pub table_name: String, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "sequence-number")] - pub sequence_number: i64, - #[serde(rename = "operation")] - pub operation: String, - #[serde(rename = "metrics")] - pub metrics: std::collections::HashMap, - #[serde(rename = "metadata", skip_serializing_if = "Option::is_none")] - pub metadata: Option>, -} - -impl CommitReport { - pub fn new( - table_name: String, - snapshot_id: i64, - sequence_number: i64, - operation: String, - metrics: std::collections::HashMap, - ) -> CommitReport { - CommitReport { - table_name, - snapshot_id, - sequence_number, - operation, - metrics, - metadata: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/commit_table_request.rs b/crates/sail-catalog-iceberg/src/models/commit_table_request.rs deleted file mode 100644 index 3fbe841f9..000000000 --- a/crates/sail-catalog-iceberg/src/models/commit_table_request.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CommitTableRequest { - /// Table identifier to update; must be present for CommitTransactionRequest - #[serde(rename = "identifier", skip_serializing_if = "Option::is_none")] - pub identifier: Option>, - #[serde(rename = "requirements")] - pub requirements: Vec, - #[serde(rename = "updates")] - pub updates: Vec, -} - -impl CommitTableRequest { - pub fn new( - requirements: Vec, - updates: Vec, - ) -> CommitTableRequest { - CommitTableRequest { - identifier: None, - requirements, - updates, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/commit_table_response.rs b/crates/sail-catalog-iceberg/src/models/commit_table_response.rs deleted file mode 100644 index 60850e740..000000000 --- a/crates/sail-catalog-iceberg/src/models/commit_table_response.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CommitTableResponse { - #[serde(rename = "metadata-location")] - pub metadata_location: String, - #[serde(rename = "metadata")] - pub metadata: Box, -} - -impl CommitTableResponse { - pub fn new(metadata_location: String, metadata: models::TableMetadata) -> CommitTableResponse { - CommitTableResponse { - metadata_location, - metadata: Box::new(metadata), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/commit_transaction_request.rs b/crates/sail-catalog-iceberg/src/models/commit_transaction_request.rs deleted file mode 100644 index 8291c9279..000000000 --- a/crates/sail-catalog-iceberg/src/models/commit_transaction_request.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CommitTransactionRequest { - #[serde(rename = "table-changes")] - pub table_changes: Vec, -} - -impl CommitTransactionRequest { - pub fn new(table_changes: Vec) -> CommitTransactionRequest { - CommitTransactionRequest { table_changes } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/commit_view_request.rs b/crates/sail-catalog-iceberg/src/models/commit_view_request.rs deleted file mode 100644 index 8796645ea..000000000 --- a/crates/sail-catalog-iceberg/src/models/commit_view_request.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CommitViewRequest { - /// View identifier to update - #[serde(rename = "identifier", skip_serializing_if = "Option::is_none")] - pub identifier: Option>, - #[serde(rename = "requirements", skip_serializing_if = "Option::is_none")] - pub requirements: Option>, - #[serde(rename = "updates")] - pub updates: Vec, -} - -impl CommitViewRequest { - pub fn new(updates: Vec) -> CommitViewRequest { - CommitViewRequest { - identifier: None, - requirements: None, - updates, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/completed_planning_result.rs b/crates/sail-catalog-iceberg/src/models/completed_planning_result.rs deleted file mode 100644 index aae34c9e0..000000000 --- a/crates/sail-catalog-iceberg/src/models/completed_planning_result.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// CompletedPlanningResult : Completed server-side planning result -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CompletedPlanningResult { - /// Delete files referenced by file scan tasks - #[serde(rename = "delete-files", skip_serializing_if = "Option::is_none")] - pub delete_files: Option>, - #[serde(rename = "file-scan-tasks", skip_serializing_if = "Option::is_none")] - pub file_scan_tasks: Option>, - #[serde(rename = "plan-tasks", skip_serializing_if = "Option::is_none")] - pub plan_tasks: Option>, - #[serde(rename = "status")] - pub status: models::PlanStatus, -} - -impl CompletedPlanningResult { - /// Completed server-side planning result - pub fn new(status: models::PlanStatus) -> CompletedPlanningResult { - CompletedPlanningResult { - delete_files: None, - file_scan_tasks: None, - plan_tasks: None, - status, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/completed_planning_with_id_result.rs b/crates/sail-catalog-iceberg/src/models/completed_planning_with_id_result.rs deleted file mode 100644 index 0a6ae84bb..000000000 --- a/crates/sail-catalog-iceberg/src/models/completed_planning_with_id_result.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CompletedPlanningWithIdResult { - /// Delete files referenced by file scan tasks - #[serde(rename = "delete-files", skip_serializing_if = "Option::is_none")] - pub delete_files: Option>, - #[serde(rename = "file-scan-tasks", skip_serializing_if = "Option::is_none")] - pub file_scan_tasks: Option>, - #[serde(rename = "plan-tasks", skip_serializing_if = "Option::is_none")] - pub plan_tasks: Option>, - #[serde(rename = "status")] - pub status: models::PlanStatus, - /// ID used to track a planning request - #[serde(rename = "plan-id", skip_serializing_if = "Option::is_none")] - pub plan_id: Option, -} - -impl CompletedPlanningWithIdResult { - pub fn new(status: models::PlanStatus) -> CompletedPlanningWithIdResult { - CompletedPlanningWithIdResult { - delete_files: None, - file_scan_tasks: None, - plan_tasks: None, - status, - plan_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/content_file.rs b/crates/sail-catalog-iceberg/src/models/content_file.rs deleted file mode 100644 index 51035c280..000000000 --- a/crates/sail-catalog-iceberg/src/models/content_file.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "content")] -pub enum ContentFile { - #[serde(rename = "data")] - DataFile { - #[serde(rename = "file-path")] - file_path: String, - #[serde(rename = "file-format")] - file_format: models::FileFormat, - #[serde(rename = "spec-id")] - spec_id: i32, - /// A list of partition field values ordered based on the fields of the partition spec specified by the `spec-id` - #[serde(rename = "partition")] - partition: Vec, - /// Total file size in bytes - #[serde(rename = "file-size-in-bytes")] - file_size_in_bytes: i64, - /// Number of records in the file - #[serde(rename = "record-count")] - record_count: i64, - /// Encryption key metadata blob - #[serde(rename = "key-metadata", skip_serializing_if = "Option::is_none")] - key_metadata: Option, - /// List of splittable offsets - #[serde(rename = "split-offsets", skip_serializing_if = "Option::is_none")] - split_offsets: Option>, - #[serde(rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - sort_order_id: Option, - }, - #[serde(rename = "equality-deletes")] - EqualityDeleteFile { - #[serde(rename = "file-path")] - file_path: String, - #[serde(rename = "file-format")] - file_format: models::FileFormat, - #[serde(rename = "spec-id")] - spec_id: i32, - /// A list of partition field values ordered based on the fields of the partition spec specified by the `spec-id` - #[serde(rename = "partition")] - partition: Vec, - /// Total file size in bytes - #[serde(rename = "file-size-in-bytes")] - file_size_in_bytes: i64, - /// Number of records in the file - #[serde(rename = "record-count")] - record_count: i64, - /// Encryption key metadata blob - #[serde(rename = "key-metadata", skip_serializing_if = "Option::is_none")] - key_metadata: Option, - /// List of splittable offsets - #[serde(rename = "split-offsets", skip_serializing_if = "Option::is_none")] - split_offsets: Option>, - #[serde(rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - sort_order_id: Option, - }, - #[serde(rename = "position-deletes")] - PositionDeleteFile { - #[serde(rename = "file-path")] - file_path: String, - #[serde(rename = "file-format")] - file_format: models::FileFormat, - #[serde(rename = "spec-id")] - spec_id: i32, - /// A list of partition field values ordered based on the fields of the partition spec specified by the `spec-id` - #[serde(rename = "partition")] - partition: Vec, - /// Total file size in bytes - #[serde(rename = "file-size-in-bytes")] - file_size_in_bytes: i64, - /// Number of records in the file - #[serde(rename = "record-count")] - record_count: i64, - /// Encryption key metadata blob - #[serde(rename = "key-metadata", skip_serializing_if = "Option::is_none")] - key_metadata: Option, - /// List of splittable offsets - #[serde(rename = "split-offsets", skip_serializing_if = "Option::is_none")] - split_offsets: Option>, - #[serde(rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - sort_order_id: Option, - }, -} - -impl Default for ContentFile { - fn default() -> Self { - Self::DataFile { - file_path: Default::default(), - file_format: Default::default(), - spec_id: Default::default(), - partition: Default::default(), - file_size_in_bytes: Default::default(), - record_count: Default::default(), - key_metadata: Default::default(), - split_offsets: Default::default(), - sort_order_id: Default::default(), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/count_map.rs b/crates/sail-catalog-iceberg/src/models/count_map.rs deleted file mode 100644 index 0601c13b7..000000000 --- a/crates/sail-catalog-iceberg/src/models/count_map.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CountMap { - /// List of integer column ids for each corresponding value - #[serde(rename = "keys", skip_serializing_if = "Option::is_none")] - pub keys: Option>, - /// List of Long values, matched to 'keys' by index - #[serde(rename = "values", skip_serializing_if = "Option::is_none")] - pub values: Option>, -} - -impl CountMap { - pub fn new() -> CountMap { - CountMap { - keys: None, - values: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/counter_result.rs b/crates/sail-catalog-iceberg/src/models/counter_result.rs deleted file mode 100644 index fc4d37146..000000000 --- a/crates/sail-catalog-iceberg/src/models/counter_result.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CounterResult { - #[serde(rename = "unit")] - pub unit: String, - #[serde(rename = "value")] - pub value: i64, -} - -impl CounterResult { - pub fn new(unit: String, value: i64) -> CounterResult { - CounterResult { unit, value } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/create_namespace_request.rs b/crates/sail-catalog-iceberg/src/models/create_namespace_request.rs deleted file mode 100644 index d7374f343..000000000 --- a/crates/sail-catalog-iceberg/src/models/create_namespace_request.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CreateNamespaceRequest { - /// Reference to one or more levels of a namespace - #[serde(rename = "namespace")] - pub namespace: Vec, - /// Configured string to string map of properties for the namespace - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl CreateNamespaceRequest { - pub fn new(namespace: Vec) -> CreateNamespaceRequest { - CreateNamespaceRequest { - namespace, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/create_namespace_response.rs b/crates/sail-catalog-iceberg/src/models/create_namespace_response.rs deleted file mode 100644 index 691d34b82..000000000 --- a/crates/sail-catalog-iceberg/src/models/create_namespace_response.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CreateNamespaceResponse { - /// Reference to one or more levels of a namespace - #[serde(rename = "namespace")] - pub namespace: Vec, - /// Properties stored on the namespace, if supported by the server. - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl CreateNamespaceResponse { - pub fn new(namespace: Vec) -> CreateNamespaceResponse { - CreateNamespaceResponse { - namespace, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/create_table_request.rs b/crates/sail-catalog-iceberg/src/models/create_table_request.rs deleted file mode 100644 index 1b7d191de..000000000 --- a/crates/sail-catalog-iceberg/src/models/create_table_request.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CreateTableRequest { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "location", skip_serializing_if = "Option::is_none")] - pub location: Option, - #[serde(rename = "schema")] - pub schema: Box, - #[serde(rename = "partition-spec", skip_serializing_if = "Option::is_none")] - pub partition_spec: Option>, - #[serde(rename = "write-order", skip_serializing_if = "Option::is_none")] - pub write_order: Option>, - #[serde(rename = "stage-create", skip_serializing_if = "Option::is_none")] - pub stage_create: Option, - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl CreateTableRequest { - pub fn new(name: String, schema: models::Schema) -> CreateTableRequest { - CreateTableRequest { - name, - location: None, - schema: Box::new(schema), - partition_spec: None, - write_order: None, - stage_create: None, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/create_view_request.rs b/crates/sail-catalog-iceberg/src/models/create_view_request.rs deleted file mode 100644 index e243fcbdf..000000000 --- a/crates/sail-catalog-iceberg/src/models/create_view_request.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct CreateViewRequest { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "location", skip_serializing_if = "Option::is_none")] - pub location: Option, - #[serde(rename = "schema")] - pub schema: Box, - /// The view version to create, will replace the schema-id sent within the view-version with the id assigned to the provided schema - #[serde(rename = "view-version")] - pub view_version: Box, - #[serde(rename = "properties")] - pub properties: std::collections::HashMap, -} - -impl CreateViewRequest { - pub fn new( - name: String, - schema: models::Schema, - view_version: models::ViewVersion, - properties: std::collections::HashMap, - ) -> CreateViewRequest { - CreateViewRequest { - name, - location: None, - schema: Box::new(schema), - view_version: Box::new(view_version), - properties, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/data_file.rs b/crates/sail-catalog-iceberg/src/models/data_file.rs deleted file mode 100644 index e97b894bb..000000000 --- a/crates/sail-catalog-iceberg/src/models/data_file.rs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct DataFile { - #[serde(rename = "content")] - pub content: String, - /// The first row ID assigned to the first row in the data file - #[serde(rename = "first-row-id", skip_serializing_if = "Option::is_none")] - pub first_row_id: Option, - /// Map of column id to total count, including null and NaN - #[serde(rename = "column-sizes", skip_serializing_if = "Option::is_none")] - pub column_sizes: Option>, - /// Map of column id to null value count - #[serde(rename = "value-counts", skip_serializing_if = "Option::is_none")] - pub value_counts: Option>, - /// Map of column id to null value count - #[serde(rename = "null-value-counts", skip_serializing_if = "Option::is_none")] - pub null_value_counts: Option>, - /// Map of column id to number of NaN values in the column - #[serde(rename = "nan-value-counts", skip_serializing_if = "Option::is_none")] - pub nan_value_counts: Option>, - /// Map of column id to lower bound primitive type values - #[serde(rename = "lower-bounds", skip_serializing_if = "Option::is_none")] - pub lower_bounds: Option>, - /// Map of column id to upper bound primitive type values - #[serde(rename = "upper-bounds", skip_serializing_if = "Option::is_none")] - pub upper_bounds: Option>, - #[serde(rename = "file-path")] - pub file_path: String, - #[serde(rename = "file-format")] - pub file_format: models::FileFormat, - #[serde(rename = "spec-id")] - pub spec_id: i32, - /// A list of partition field values ordered based on the fields of the partition spec specified by the `spec-id` - #[serde(rename = "partition")] - pub partition: Vec, - /// Total file size in bytes - #[serde(rename = "file-size-in-bytes")] - pub file_size_in_bytes: i64, - /// Number of records in the file - #[serde(rename = "record-count")] - pub record_count: i64, - /// Encryption key metadata blob - #[serde(rename = "key-metadata", skip_serializing_if = "Option::is_none")] - pub key_metadata: Option, - /// List of splittable offsets - #[serde(rename = "split-offsets", skip_serializing_if = "Option::is_none")] - pub split_offsets: Option>, - #[serde(rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - pub sort_order_id: Option, -} - -impl DataFile { - pub fn new( - content: String, - file_path: String, - file_format: models::FileFormat, - spec_id: i32, - partition: Vec, - file_size_in_bytes: i64, - record_count: i64, - ) -> DataFile { - DataFile { - content, - first_row_id: None, - column_sizes: None, - value_counts: None, - null_value_counts: None, - nan_value_counts: None, - lower_bounds: None, - upper_bounds: None, - file_path, - file_format, - spec_id, - partition, - file_size_in_bytes, - record_count, - key_metadata: None, - split_offsets: None, - sort_order_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/delete_file.rs b/crates/sail-catalog-iceberg/src/models/delete_file.rs deleted file mode 100644 index 627124f4a..000000000 --- a/crates/sail-catalog-iceberg/src/models/delete_file.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "content")] -pub enum DeleteFile { - #[serde(rename = "position-deletes")] - PositionDeletes(Box), - #[serde(rename = "equality-deletes")] - EqualityDeletes(Box), -} - -impl Default for DeleteFile { - fn default() -> Self { - Self::PositionDeletes(Default::default()) - } -} diff --git a/crates/sail-catalog-iceberg/src/models/empty_planning_result.rs b/crates/sail-catalog-iceberg/src/models/empty_planning_result.rs deleted file mode 100644 index 691f22b25..000000000 --- a/crates/sail-catalog-iceberg/src/models/empty_planning_result.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// EmptyPlanningResult : Empty server-side planning result -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct EmptyPlanningResult { - #[serde(rename = "status")] - pub status: Status, -} - -impl EmptyPlanningResult { - /// Empty server-side planning result - pub fn new(status: Status) -> EmptyPlanningResult { - EmptyPlanningResult { status } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Status { - #[serde(rename = "completed")] - Completed, - #[serde(rename = "submitted")] - Submitted, - #[serde(rename = "cancelled")] - Cancelled, - #[serde(rename = "failed")] - Failed, -} - -impl Default for Status { - fn default() -> Status { - Self::Completed - } -} diff --git a/crates/sail-catalog-iceberg/src/models/encrypted_key.rs b/crates/sail-catalog-iceberg/src/models/encrypted_key.rs deleted file mode 100644 index f61facc0e..000000000 --- a/crates/sail-catalog-iceberg/src/models/encrypted_key.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; -use serde_with::serde_as; - -use crate::models; - -#[serde_as] -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct EncryptedKey { - #[serde(rename = "key-id")] - pub key_id: String, - #[serde_as(as = "serde_with::base64::Base64")] - #[serde(rename = "encrypted-key-metadata")] - pub encrypted_key_metadata: Vec, - #[serde(rename = "encrypted-by-id", skip_serializing_if = "Option::is_none")] - pub encrypted_by_id: Option, - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl EncryptedKey { - pub fn new(key_id: String, encrypted_key_metadata: Vec) -> EncryptedKey { - EncryptedKey { - key_id, - encrypted_key_metadata, - encrypted_by_id: None, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/equality_delete_file.rs b/crates/sail-catalog-iceberg/src/models/equality_delete_file.rs deleted file mode 100644 index 528c65cfa..000000000 --- a/crates/sail-catalog-iceberg/src/models/equality_delete_file.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct EqualityDeleteFile { - #[serde(rename = "content")] - pub content: String, - /// List of equality field IDs - #[serde(rename = "equality-ids", skip_serializing_if = "Option::is_none")] - pub equality_ids: Option>, - #[serde(rename = "file-path")] - pub file_path: String, - #[serde(rename = "file-format")] - pub file_format: models::FileFormat, - #[serde(rename = "spec-id")] - pub spec_id: i32, - /// A list of partition field values ordered based on the fields of the partition spec specified by the `spec-id` - #[serde(rename = "partition")] - pub partition: Vec, - /// Total file size in bytes - #[serde(rename = "file-size-in-bytes")] - pub file_size_in_bytes: i64, - /// Number of records in the file - #[serde(rename = "record-count")] - pub record_count: i64, - /// Encryption key metadata blob - #[serde(rename = "key-metadata", skip_serializing_if = "Option::is_none")] - pub key_metadata: Option, - /// List of splittable offsets - #[serde(rename = "split-offsets", skip_serializing_if = "Option::is_none")] - pub split_offsets: Option>, - #[serde(rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - pub sort_order_id: Option, -} - -impl EqualityDeleteFile { - pub fn new( - content: String, - file_path: String, - file_format: models::FileFormat, - spec_id: i32, - partition: Vec, - file_size_in_bytes: i64, - record_count: i64, - ) -> EqualityDeleteFile { - EqualityDeleteFile { - content, - equality_ids: None, - file_path, - file_format, - spec_id, - partition, - file_size_in_bytes, - record_count, - key_metadata: None, - split_offsets: None, - sort_order_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/error_model.rs b/crates/sail-catalog-iceberg/src/models/error_model.rs deleted file mode 100644 index 07a1fde81..000000000 --- a/crates/sail-catalog-iceberg/src/models/error_model.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// ErrorModel : JSON error payload returned in a response with further details on the error -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ErrorModel { - /// Human-readable error message - #[serde(rename = "message")] - pub message: String, - /// Internal type definition of the error - #[serde(rename = "type")] - pub r#type: String, - /// HTTP response code - #[serde(rename = "code")] - pub code: i32, - #[serde(rename = "stack", skip_serializing_if = "Option::is_none")] - pub stack: Option>, -} - -impl ErrorModel { - /// JSON error payload returned in a response with further details on the error - pub fn new(message: String, r#type: String, code: i32) -> ErrorModel { - ErrorModel { - message, - r#type, - code, - stack: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/expression.rs b/crates/sail-catalog-iceberg/src/models/expression.rs deleted file mode 100644 index 46e9db3fe..000000000 --- a/crates/sail-catalog-iceberg/src/models/expression.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum Expression { - TrueExpression(Box), - FalseExpression(Box), - AndOrExpression(Box), - NotExpression(Box), - SetExpression(Box), - LiteralExpression(Box), - UnaryExpression(Box), -} - -impl Default for Expression { - fn default() -> Self { - Self::TrueExpression(Default::default()) - } -} diff --git a/crates/sail-catalog-iceberg/src/models/failed_planning_result.rs b/crates/sail-catalog-iceberg/src/models/failed_planning_result.rs deleted file mode 100644 index eca927e96..000000000 --- a/crates/sail-catalog-iceberg/src/models/failed_planning_result.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// FailedPlanningResult : Failed server-side planning result -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FailedPlanningResult { - #[serde(rename = "error")] - pub error: Box, - #[serde(rename = "status")] - pub status: models::PlanStatus, -} - -impl FailedPlanningResult { - /// Failed server-side planning result - pub fn new(error: models::ErrorModel, status: models::PlanStatus) -> FailedPlanningResult { - FailedPlanningResult { - error: Box::new(error), - status, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/false_expression.rs b/crates/sail-catalog-iceberg/src/models/false_expression.rs deleted file mode 100644 index f7231d3cb..000000000 --- a/crates/sail-catalog-iceberg/src/models/false_expression.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FalseExpression { - #[serde(rename = "type")] - pub r#type: String, -} - -impl FalseExpression { - pub fn new(r#type: String) -> FalseExpression { - FalseExpression { r#type } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/fetch_planning_result.rs b/crates/sail-catalog-iceberg/src/models/fetch_planning_result.rs deleted file mode 100644 index dbcc3da63..000000000 --- a/crates/sail-catalog-iceberg/src/models/fetch_planning_result.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// FetchPlanningResult : Result of server-side scan planning for fetchPlanningResult -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "status")] -pub enum FetchPlanningResult { - #[serde(rename = "completed")] - Completed(Box), - #[serde(rename = "failed")] - Failed(Box), - #[serde(rename = "submitted")] - Submitted(Box), -} - -impl Default for FetchPlanningResult { - fn default() -> Self { - Self::Completed(Default::default()) - } -} diff --git a/crates/sail-catalog-iceberg/src/models/fetch_scan_tasks_request.rs b/crates/sail-catalog-iceberg/src/models/fetch_scan_tasks_request.rs deleted file mode 100644 index 7034896e1..000000000 --- a/crates/sail-catalog-iceberg/src/models/fetch_scan_tasks_request.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FetchScanTasksRequest { - /// An opaque string provided by the REST server that represents a unit of work to produce file scan tasks for scan planning. This allows clients to fetch tasks across multiple requests to accommodate large result sets. - #[serde(rename = "plan-task")] - pub plan_task: String, -} - -impl FetchScanTasksRequest { - pub fn new(plan_task: String) -> FetchScanTasksRequest { - FetchScanTasksRequest { plan_task } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/fetch_scan_tasks_result.rs b/crates/sail-catalog-iceberg/src/models/fetch_scan_tasks_result.rs deleted file mode 100644 index b28891d18..000000000 --- a/crates/sail-catalog-iceberg/src/models/fetch_scan_tasks_result.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// FetchScanTasksResult : Response schema for fetchScanTasks -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FetchScanTasksResult { - /// Delete files referenced by file scan tasks - #[serde(rename = "delete-files", skip_serializing_if = "Option::is_none")] - pub delete_files: Option>, - #[serde(rename = "file-scan-tasks", skip_serializing_if = "Option::is_none")] - pub file_scan_tasks: Option>, - #[serde(rename = "plan-tasks", skip_serializing_if = "Option::is_none")] - pub plan_tasks: Option>, -} - -impl FetchScanTasksResult { - /// Response schema for fetchScanTasks - pub fn new() -> FetchScanTasksResult { - FetchScanTasksResult { - delete_files: None, - file_scan_tasks: None, - plan_tasks: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/file_format.rs b/crates/sail-catalog-iceberg/src/models/file_format.rs deleted file mode 100644 index ee1a34242..000000000 --- a/crates/sail-catalog-iceberg/src/models/file_format.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum FileFormat { - #[serde(rename = "avro")] - Avro, - #[serde(rename = "orc")] - Orc, - #[serde(rename = "parquet")] - Parquet, - #[serde(rename = "puffin")] - Puffin, -} - -impl std::fmt::Display for FileFormat { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Avro => write!(f, "avro"), - Self::Orc => write!(f, "orc"), - Self::Parquet => write!(f, "parquet"), - Self::Puffin => write!(f, "puffin"), - } - } -} - -impl Default for FileFormat { - fn default() -> FileFormat { - Self::Avro - } -} diff --git a/crates/sail-catalog-iceberg/src/models/file_scan_task.rs b/crates/sail-catalog-iceberg/src/models/file_scan_task.rs deleted file mode 100644 index ba7ca784b..000000000 --- a/crates/sail-catalog-iceberg/src/models/file_scan_task.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FileScanTask { - #[serde(rename = "data-file")] - pub data_file: Box, - /// A list of indices in the delete files array (0-based) - #[serde( - rename = "delete-file-references", - skip_serializing_if = "Option::is_none" - )] - pub delete_file_references: Option>, - /// An optional filter to be applied to rows in this file scan task. If the residual is not present, the client must produce the residual or use the original filter. - #[serde(rename = "residual-filter", skip_serializing_if = "Option::is_none")] - pub residual_filter: Option>, -} - -impl FileScanTask { - pub fn new(data_file: models::DataFile) -> FileScanTask { - FileScanTask { - data_file: Box::new(data_file), - delete_file_references: None, - residual_filter: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/get_namespace_response.rs b/crates/sail-catalog-iceberg/src/models/get_namespace_response.rs deleted file mode 100644 index a4e528a18..000000000 --- a/crates/sail-catalog-iceberg/src/models/get_namespace_response.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct GetNamespaceResponse { - /// Reference to one or more levels of a namespace - #[serde(rename = "namespace")] - pub namespace: Vec, - /// Properties stored on the namespace, if supported by the server. If the server does not support namespace properties, it should return null for this field. If namespace properties are supported, but none are set, it should return an empty object. - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl GetNamespaceResponse { - pub fn new(namespace: Vec) -> GetNamespaceResponse { - GetNamespaceResponse { - namespace, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/iceberg_error_response.rs b/crates/sail-catalog-iceberg/src/models/iceberg_error_response.rs deleted file mode 100644 index 15f2c25b7..000000000 --- a/crates/sail-catalog-iceberg/src/models/iceberg_error_response.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// IcebergErrorResponse : JSON wrapper for all error responses (non-2xx) -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct IcebergErrorResponse { - #[serde(rename = "error")] - pub error: Box, -} - -impl IcebergErrorResponse { - /// JSON wrapper for all error responses (non-2xx) - pub fn new(error: models::ErrorModel) -> IcebergErrorResponse { - IcebergErrorResponse { - error: Box::new(error), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/list_namespaces_response.rs b/crates/sail-catalog-iceberg/src/models/list_namespaces_response.rs deleted file mode 100644 index 77eef1b1e..000000000 --- a/crates/sail-catalog-iceberg/src/models/list_namespaces_response.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ListNamespacesResponse { - /// An opaque token that allows clients to make use of pagination for list APIs (e.g. ListTables). Clients may initiate the first paginated request by sending an empty query parameter `pageToken` to the server. Servers that support pagination should identify the `pageToken` parameter and return a `next-page-token` in the response if there are more results available. After the initial request, the value of `next-page-token` from each response must be used as the `pageToken` parameter value for the next request. The server must return `null` value for the `next-page-token` in the last response. Servers that support pagination must return all results in a single response with the value of `next-page-token` set to `null` if the query parameter `pageToken` is not set in the request. Servers that do not support pagination should ignore the `pageToken` parameter and return all results in a single response. The `next-page-token` must be omitted from the response. Clients must interpret either `null` or missing response value of `next-page-token` as the end of the listing results. - #[serde(rename = "next-page-token", skip_serializing_if = "Option::is_none")] - pub next_page_token: Option, - #[serde(rename = "namespaces", skip_serializing_if = "Option::is_none")] - pub namespaces: Option>>, -} - -impl ListNamespacesResponse { - pub fn new() -> ListNamespacesResponse { - ListNamespacesResponse { - next_page_token: None, - namespaces: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/list_tables_response.rs b/crates/sail-catalog-iceberg/src/models/list_tables_response.rs deleted file mode 100644 index 4a395156b..000000000 --- a/crates/sail-catalog-iceberg/src/models/list_tables_response.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ListTablesResponse { - /// An opaque token that allows clients to make use of pagination for list APIs (e.g. ListTables). Clients may initiate the first paginated request by sending an empty query parameter `pageToken` to the server. Servers that support pagination should identify the `pageToken` parameter and return a `next-page-token` in the response if there are more results available. After the initial request, the value of `next-page-token` from each response must be used as the `pageToken` parameter value for the next request. The server must return `null` value for the `next-page-token` in the last response. Servers that support pagination must return all results in a single response with the value of `next-page-token` set to `null` if the query parameter `pageToken` is not set in the request. Servers that do not support pagination should ignore the `pageToken` parameter and return all results in a single response. The `next-page-token` must be omitted from the response. Clients must interpret either `null` or missing response value of `next-page-token` as the end of the listing results. - #[serde(rename = "next-page-token", skip_serializing_if = "Option::is_none")] - pub next_page_token: Option, - #[serde(rename = "identifiers", skip_serializing_if = "Option::is_none")] - pub identifiers: Option>, -} - -impl ListTablesResponse { - pub fn new() -> ListTablesResponse { - ListTablesResponse { - next_page_token: None, - identifiers: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/literal_expression.rs b/crates/sail-catalog-iceberg/src/models/literal_expression.rs deleted file mode 100644 index f8e7659ae..000000000 --- a/crates/sail-catalog-iceberg/src/models/literal_expression.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct LiteralExpression { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "term")] - pub term: Box, - #[serde(rename = "value")] - pub value: serde_json::Value, -} - -impl LiteralExpression { - pub fn new(r#type: String, term: models::Term, value: serde_json::Value) -> LiteralExpression { - LiteralExpression { - r#type, - term: Box::new(term), - value, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/load_credentials_response.rs b/crates/sail-catalog-iceberg/src/models/load_credentials_response.rs deleted file mode 100644 index 398200469..000000000 --- a/crates/sail-catalog-iceberg/src/models/load_credentials_response.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct LoadCredentialsResponse { - #[serde(rename = "storage-credentials")] - pub storage_credentials: Vec, -} - -impl LoadCredentialsResponse { - pub fn new(storage_credentials: Vec) -> LoadCredentialsResponse { - LoadCredentialsResponse { - storage_credentials, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/load_table_result.rs b/crates/sail-catalog-iceberg/src/models/load_table_result.rs deleted file mode 100644 index e12da58d3..000000000 --- a/crates/sail-catalog-iceberg/src/models/load_table_result.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// LoadTableResult : Result used when a table is successfully loaded. The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. Clients can check whether metadata has changed by comparing metadata locations after the table has been created. The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. The following configurations should be respected by clients: ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled ## AWS Configurations The following configurations should be respected when working with tables stored in AWS S3 - `client.region`: region to configure client for making requests to AWS - `s3.access-key-id`: id for credentials that provide access to the data in S3 - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled ## Storage Credentials Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct LoadTableResult { - /// May be null if the table is staged as part of a transaction - #[serde(rename = "metadata-location", skip_serializing_if = "Option::is_none")] - pub metadata_location: Option, - #[serde(rename = "metadata")] - pub metadata: Box, - #[serde(rename = "config", skip_serializing_if = "Option::is_none")] - pub config: Option>, - #[serde( - rename = "storage-credentials", - skip_serializing_if = "Option::is_none" - )] - pub storage_credentials: Option>, -} - -impl LoadTableResult { - /// Result used when a table is successfully loaded. The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. Clients can check whether metadata has changed by comparing metadata locations after the table has been created. The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. The following configurations should be respected by clients: ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled ## AWS Configurations The following configurations should be respected when working with tables stored in AWS S3 - `client.region`: region to configure client for making requests to AWS - `s3.access-key-id`: id for credentials that provide access to the data in S3 - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled ## Storage Credentials Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. - pub fn new(metadata: models::TableMetadata) -> LoadTableResult { - LoadTableResult { - metadata_location: None, - metadata: Box::new(metadata), - config: None, - storage_credentials: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/load_view_result.rs b/crates/sail-catalog-iceberg/src/models/load_view_result.rs deleted file mode 100644 index aa86249ea..000000000 --- a/crates/sail-catalog-iceberg/src/models/load_view_result.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// LoadViewResult : Result used when a view is successfully loaded. The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. Clients can check whether metadata has changed by comparing metadata locations after the view has been created. The `config` map returns view-specific configuration for the view's resources. The following configurations should be respected by clients: ## General Configurations - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct LoadViewResult { - #[serde(rename = "metadata-location")] - pub metadata_location: String, - #[serde(rename = "metadata")] - pub metadata: Box, - #[serde(rename = "config", skip_serializing_if = "Option::is_none")] - pub config: Option>, -} - -impl LoadViewResult { - /// Result used when a view is successfully loaded. The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. Clients can check whether metadata has changed by comparing metadata locations after the view has been created. The `config` map returns view-specific configuration for the view's resources. The following configurations should be respected by clients: ## General Configurations - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled - pub fn new(metadata_location: String, metadata: models::ViewMetadata) -> LoadViewResult { - LoadViewResult { - metadata_location, - metadata: Box::new(metadata), - config: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/metadata_log_inner.rs b/crates/sail-catalog-iceberg/src/models/metadata_log_inner.rs deleted file mode 100644 index 19851eb73..000000000 --- a/crates/sail-catalog-iceberg/src/models/metadata_log_inner.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct MetadataLogInner { - #[serde(rename = "metadata-file")] - pub metadata_file: String, - #[serde(rename = "timestamp-ms")] - pub timestamp_ms: i64, -} - -impl MetadataLogInner { - pub fn new(metadata_file: String, timestamp_ms: i64) -> MetadataLogInner { - MetadataLogInner { - metadata_file, - timestamp_ms, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/metric_result.rs b/crates/sail-catalog-iceberg/src/models/metric_result.rs deleted file mode 100644 index 97f2ea906..000000000 --- a/crates/sail-catalog-iceberg/src/models/metric_result.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct MetricResult { - #[serde(rename = "unit")] - pub unit: String, - #[serde(rename = "value")] - pub value: i64, - #[serde(rename = "time-unit")] - pub time_unit: String, - #[serde(rename = "count")] - pub count: i64, - #[serde(rename = "total-duration")] - pub total_duration: i64, -} - -impl MetricResult { - pub fn new( - unit: String, - value: i64, - time_unit: String, - count: i64, - total_duration: i64, - ) -> MetricResult { - MetricResult { - unit, - value, - time_unit, - count, - total_duration, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/mod.rs b/crates/sail-catalog-iceberg/src/models/mod.rs deleted file mode 100644 index 9e6872d4f..000000000 --- a/crates/sail-catalog-iceberg/src/models/mod.rs +++ /dev/null @@ -1,258 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod add_encryption_key_update; -pub use self::add_encryption_key_update::AddEncryptionKeyUpdate; -pub mod add_partition_spec_update; -pub use self::add_partition_spec_update::AddPartitionSpecUpdate; -pub mod add_schema_update; -pub use self::add_schema_update::AddSchemaUpdate; -pub mod add_snapshot_update; -pub use self::add_snapshot_update::AddSnapshotUpdate; -pub mod add_sort_order_update; -pub use self::add_sort_order_update::AddSortOrderUpdate; -pub mod add_view_version_update; -pub use self::add_view_version_update::AddViewVersionUpdate; -pub mod and_or_expression; -pub use self::and_or_expression::AndOrExpression; -pub mod assert_create; -pub use self::assert_create::AssertCreate; -pub mod assert_current_schema_id; -pub use self::assert_current_schema_id::AssertCurrentSchemaId; -pub mod assert_default_sort_order_id; -pub use self::assert_default_sort_order_id::AssertDefaultSortOrderId; -pub mod assert_default_spec_id; -pub use self::assert_default_spec_id::AssertDefaultSpecId; -pub mod assert_last_assigned_field_id; -pub use self::assert_last_assigned_field_id::AssertLastAssignedFieldId; -pub mod assert_last_assigned_partition_id; -pub use self::assert_last_assigned_partition_id::AssertLastAssignedPartitionId; -pub mod assert_ref_snapshot_id; -pub use self::assert_ref_snapshot_id::AssertRefSnapshotId; -pub mod assert_table_uuid; -pub use self::assert_table_uuid::AssertTableUuid; -pub mod assert_view_uuid; -pub use self::assert_view_uuid::AssertViewUuid; -pub mod assign_uuid_update; -pub use self::assign_uuid_update::AssignUuidUpdate; -pub mod async_planning_result; -pub use self::async_planning_result::AsyncPlanningResult; -pub mod base_update; -pub use self::base_update::BaseUpdate; -pub mod blob_metadata; -pub use self::blob_metadata::BlobMetadata; -pub mod catalog_config; -pub use self::catalog_config::CatalogConfig; -pub mod commit_report; -pub use self::commit_report::CommitReport; -pub mod commit_table_request; -pub use self::commit_table_request::CommitTableRequest; -pub mod commit_table_response; -pub use self::commit_table_response::CommitTableResponse; -pub mod commit_transaction_request; -pub use self::commit_transaction_request::CommitTransactionRequest; -pub mod commit_view_request; -pub use self::commit_view_request::CommitViewRequest; -pub mod completed_planning_result; -pub use self::completed_planning_result::CompletedPlanningResult; -pub mod completed_planning_with_id_result; -pub use self::completed_planning_with_id_result::CompletedPlanningWithIdResult; -pub mod content_file; -pub use self::content_file::ContentFile; -pub mod count_map; -pub use self::count_map::CountMap; -pub mod counter_result; -pub use self::counter_result::CounterResult; -pub mod create_namespace_request; -pub use self::create_namespace_request::CreateNamespaceRequest; -pub mod create_namespace_response; -pub use self::create_namespace_response::CreateNamespaceResponse; -pub mod create_table_request; -pub use self::create_table_request::CreateTableRequest; -pub mod create_view_request; -pub use self::create_view_request::CreateViewRequest; -pub mod data_file; -pub use self::data_file::DataFile; -pub mod delete_file; -pub use self::delete_file::DeleteFile; -pub mod empty_planning_result; -pub use self::empty_planning_result::EmptyPlanningResult; -pub mod encrypted_key; -pub use self::encrypted_key::EncryptedKey; -pub mod equality_delete_file; -pub use self::equality_delete_file::EqualityDeleteFile; -pub mod error_model; -pub use self::error_model::ErrorModel; -pub mod expression; -pub use self::expression::Expression; -pub mod failed_planning_result; -pub use self::failed_planning_result::FailedPlanningResult; -pub mod false_expression; -pub use self::false_expression::FalseExpression; -pub mod fetch_planning_result; -pub use self::fetch_planning_result::FetchPlanningResult; -pub mod fetch_scan_tasks_request; -pub use self::fetch_scan_tasks_request::FetchScanTasksRequest; -pub mod fetch_scan_tasks_result; -pub use self::fetch_scan_tasks_result::FetchScanTasksResult; -pub mod file_format; -pub use self::file_format::FileFormat; -pub mod file_scan_task; -pub use self::file_scan_task::FileScanTask; -pub mod get_namespace_response; -pub use self::get_namespace_response::GetNamespaceResponse; -pub mod iceberg_error_response; -pub use self::iceberg_error_response::IcebergErrorResponse; -pub mod list_namespaces_response; -pub use self::list_namespaces_response::ListNamespacesResponse; -pub mod list_tables_response; -pub use self::list_tables_response::ListTablesResponse; -pub mod literal_expression; -pub use self::literal_expression::LiteralExpression; -pub mod load_credentials_response; -pub use self::load_credentials_response::LoadCredentialsResponse; -pub mod load_table_result; -pub use self::load_table_result::LoadTableResult; -pub mod load_view_result; -pub use self::load_view_result::LoadViewResult; -pub mod metadata_log_inner; -pub use self::metadata_log_inner::MetadataLogInner; -pub mod metric_result; -pub use self::metric_result::MetricResult; -pub mod not_expression; -pub use self::not_expression::NotExpression; -pub mod null_order; -pub use self::null_order::NullOrder; -pub mod o_auth_error; -pub use self::o_auth_error::OAuthError; -pub mod o_auth_token_response; -pub use self::o_auth_token_response::OAuthTokenResponse; -pub mod partition_field; -pub use self::partition_field::PartitionField; -pub mod partition_spec; -pub use self::partition_spec::PartitionSpec; -pub mod partition_statistics_file; -pub use self::partition_statistics_file::PartitionStatisticsFile; -pub mod plan_status; -pub use self::plan_status::PlanStatus; -pub mod plan_table_scan_request; -pub use self::plan_table_scan_request::PlanTableScanRequest; -pub mod plan_table_scan_result; -pub use self::plan_table_scan_result::PlanTableScanResult; -pub mod position_delete_file; -pub use self::position_delete_file::PositionDeleteFile; -pub mod primitive_type_value; -pub use self::primitive_type_value::PrimitiveTypeValue; -pub mod register_table_request; -pub use self::register_table_request::RegisterTableRequest; -pub mod remove_encryption_key_update; -pub use self::remove_encryption_key_update::RemoveEncryptionKeyUpdate; -pub mod remove_partition_specs_update; -pub use self::remove_partition_specs_update::RemovePartitionSpecsUpdate; -pub mod remove_partition_statistics_update; -pub use self::remove_partition_statistics_update::RemovePartitionStatisticsUpdate; -pub mod remove_properties_update; -pub use self::remove_properties_update::RemovePropertiesUpdate; -pub mod remove_schemas_update; -pub use self::remove_schemas_update::RemoveSchemasUpdate; -pub mod remove_snapshot_ref_update; -pub use self::remove_snapshot_ref_update::RemoveSnapshotRefUpdate; -pub mod remove_snapshots_update; -pub use self::remove_snapshots_update::RemoveSnapshotsUpdate; -pub mod remove_statistics_update; -pub use self::remove_statistics_update::RemoveStatisticsUpdate; -pub mod rename_table_request; -pub use self::rename_table_request::RenameTableRequest; -pub mod report_metrics_request; -pub use self::report_metrics_request::ReportMetricsRequest; -pub mod scan_report; -pub use self::scan_report::ScanReport; -pub mod scan_tasks; -pub use self::scan_tasks::ScanTasks; -pub mod schema; -pub use self::schema::Schema; -pub mod set_current_schema_update; -pub use self::set_current_schema_update::SetCurrentSchemaUpdate; -pub mod set_current_view_version_update; -pub use self::set_current_view_version_update::SetCurrentViewVersionUpdate; -pub mod set_default_sort_order_update; -pub use self::set_default_sort_order_update::SetDefaultSortOrderUpdate; -pub mod set_default_spec_update; -pub use self::set_default_spec_update::SetDefaultSpecUpdate; -pub mod set_expression; -pub use self::set_expression::SetExpression; -pub mod set_location_update; -pub use self::set_location_update::SetLocationUpdate; -pub mod set_partition_statistics_update; -pub use self::set_partition_statistics_update::SetPartitionStatisticsUpdate; -pub mod set_properties_update; -pub use self::set_properties_update::SetPropertiesUpdate; -pub mod set_snapshot_ref_update; -pub use self::set_snapshot_ref_update::SetSnapshotRefUpdate; -pub mod set_statistics_update; -pub use self::set_statistics_update::SetStatisticsUpdate; -pub mod snapshot; -pub use self::snapshot::Snapshot; -pub mod snapshot_log_inner; -pub use self::snapshot_log_inner::SnapshotLogInner; -pub mod snapshot_reference; -pub use self::snapshot_reference::SnapshotReference; -pub mod snapshot_summary; -pub use self::snapshot_summary::SnapshotSummary; -pub mod sort_direction; -pub use self::sort_direction::SortDirection; -pub mod sort_field; -pub use self::sort_field::SortField; -pub mod sort_order; -pub use self::sort_order::SortOrder; -pub mod sql_view_representation; -pub use self::sql_view_representation::SqlViewRepresentation; -pub mod statistics_file; -pub use self::statistics_file::StatisticsFile; -pub mod storage_credential; -pub use self::storage_credential::StorageCredential; -pub mod table_identifier; -pub use self::table_identifier::TableIdentifier; -pub mod table_metadata; -pub use self::table_metadata::TableMetadata; -pub mod table_requirement; -pub use self::table_requirement::TableRequirement; -pub mod table_update; -pub use self::table_update::TableUpdate; -pub mod term; -pub use self::term::Term; -pub mod timer_result; -pub use self::timer_result::TimerResult; -pub mod token_type; -pub use self::token_type::TokenType; -pub mod transform_term; -pub use self::transform_term::TransformTerm; -pub mod true_expression; -pub use self::true_expression::TrueExpression; -pub mod unary_expression; -pub use self::unary_expression::UnaryExpression; -pub mod update_namespace_properties_request; -pub use self::update_namespace_properties_request::UpdateNamespacePropertiesRequest; -pub mod update_namespace_properties_response; -pub use self::update_namespace_properties_response::UpdateNamespacePropertiesResponse; -pub mod upgrade_format_version_update; -pub use self::upgrade_format_version_update::UpgradeFormatVersionUpdate; -pub mod value_map; -pub use self::value_map::ValueMap; -pub mod view_history_entry; -pub use self::view_history_entry::ViewHistoryEntry; -pub mod view_metadata; -pub use self::view_metadata::ViewMetadata; -pub mod view_update; -pub use self::view_update::ViewUpdate; -pub mod view_version; -pub use self::view_version::ViewVersion; diff --git a/crates/sail-catalog-iceberg/src/models/not_expression.rs b/crates/sail-catalog-iceberg/src/models/not_expression.rs deleted file mode 100644 index ca7328959..000000000 --- a/crates/sail-catalog-iceberg/src/models/not_expression.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct NotExpression { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "child")] - pub child: Box, -} - -impl NotExpression { - pub fn new(r#type: String, child: models::Expression) -> NotExpression { - NotExpression { - r#type, - child: Box::new(child), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/null_order.rs b/crates/sail-catalog-iceberg/src/models/null_order.rs deleted file mode 100644 index e956fe6f6..000000000 --- a/crates/sail-catalog-iceberg/src/models/null_order.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum NullOrder { - #[serde(rename = "nulls-first")] - NullsFirst, - #[serde(rename = "nulls-last")] - NullsLast, -} - -impl std::fmt::Display for NullOrder { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::NullsFirst => write!(f, "nulls-first"), - Self::NullsLast => write!(f, "nulls-last"), - } - } -} - -impl Default for NullOrder { - fn default() -> NullOrder { - Self::NullsFirst - } -} diff --git a/crates/sail-catalog-iceberg/src/models/o_auth_error.rs b/crates/sail-catalog-iceberg/src/models/o_auth_error.rs deleted file mode 100644 index 3ed815812..000000000 --- a/crates/sail-catalog-iceberg/src/models/o_auth_error.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// OAuthError : The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this spec, see description of the endpoint. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct OAuthError { - #[serde(rename = "error")] - pub error: Error, - #[serde(rename = "error_description", skip_serializing_if = "Option::is_none")] - pub error_description: Option, - #[serde(rename = "error_uri", skip_serializing_if = "Option::is_none")] - pub error_uri: Option, -} - -impl OAuthError { - /// The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this spec, see description of the endpoint. - pub fn new(error: Error) -> OAuthError { - OAuthError { - error, - error_description: None, - error_uri: None, - } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Error { - #[serde(rename = "invalid_request")] - InvalidRequest, - #[serde(rename = "invalid_client")] - InvalidClient, - #[serde(rename = "invalid_grant")] - InvalidGrant, - #[serde(rename = "unauthorized_client")] - UnauthorizedClient, - #[serde(rename = "unsupported_grant_type")] - UnsupportedGrantType, - #[serde(rename = "invalid_scope")] - InvalidScope, -} - -impl Default for Error { - fn default() -> Error { - Self::InvalidRequest - } -} diff --git a/crates/sail-catalog-iceberg/src/models/o_auth_token_response.rs b/crates/sail-catalog-iceberg/src/models/o_auth_token_response.rs deleted file mode 100644 index 81ccf68c7..000000000 --- a/crates/sail-catalog-iceberg/src/models/o_auth_token_response.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// OAuthTokenResponse : The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this spec, see description of the endpoint. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct OAuthTokenResponse { - /// The access token, for client credentials or token exchange - #[serde(rename = "access_token")] - pub access_token: String, - /// Access token type for client credentials or token exchange See https://datatracker.ietf.org/doc/html/rfc6749#section-7.1 - #[serde(rename = "token_type")] - pub token_type: TokenType, - /// Lifetime of the access token in seconds for client credentials or token exchange - #[serde(rename = "expires_in", skip_serializing_if = "Option::is_none")] - pub expires_in: Option, - #[serde(rename = "issued_token_type", skip_serializing_if = "Option::is_none")] - pub issued_token_type: Option, - /// Refresh token for client credentials or token exchange - #[serde(rename = "refresh_token", skip_serializing_if = "Option::is_none")] - pub refresh_token: Option, - /// Authorization scope for client credentials or token exchange - #[serde(rename = "scope", skip_serializing_if = "Option::is_none")] - pub scope: Option, -} - -impl OAuthTokenResponse { - /// The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this spec, see description of the endpoint. - pub fn new(access_token: String, token_type: TokenType) -> OAuthTokenResponse { - OAuthTokenResponse { - access_token, - token_type, - expires_in: None, - issued_token_type: None, - refresh_token: None, - scope: None, - } - } -} -/// Access token type for client credentials or token exchange See https://datatracker.ietf.org/doc/html/rfc6749#section-7.1 -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum TokenType { - #[serde(rename = "bearer")] - Bearer, - #[serde(rename = "mac")] - Mac, - #[serde(rename = "N_A")] - NA, -} - -impl Default for TokenType { - fn default() -> TokenType { - Self::Bearer - } -} diff --git a/crates/sail-catalog-iceberg/src/models/partition_field.rs b/crates/sail-catalog-iceberg/src/models/partition_field.rs deleted file mode 100644 index 9ca737725..000000000 --- a/crates/sail-catalog-iceberg/src/models/partition_field.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct PartitionField { - #[serde(rename = "field-id", skip_serializing_if = "Option::is_none")] - pub field_id: Option, - #[serde(rename = "source-id")] - pub source_id: i32, - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "transform")] - pub transform: String, -} - -impl PartitionField { - pub fn new(source_id: i32, name: String, transform: String) -> PartitionField { - PartitionField { - field_id: None, - source_id, - name, - transform, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/partition_spec.rs b/crates/sail-catalog-iceberg/src/models/partition_spec.rs deleted file mode 100644 index 11d93930a..000000000 --- a/crates/sail-catalog-iceberg/src/models/partition_spec.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct PartitionSpec { - #[serde(rename = "spec-id", skip_serializing_if = "Option::is_none")] - pub spec_id: Option, - #[serde(rename = "fields")] - pub fields: Vec, -} - -impl PartitionSpec { - pub fn new(fields: Vec) -> PartitionSpec { - PartitionSpec { - spec_id: None, - fields, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/partition_statistics_file.rs b/crates/sail-catalog-iceberg/src/models/partition_statistics_file.rs deleted file mode 100644 index 700aa343e..000000000 --- a/crates/sail-catalog-iceberg/src/models/partition_statistics_file.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct PartitionStatisticsFile { - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "statistics-path")] - pub statistics_path: String, - #[serde(rename = "file-size-in-bytes")] - pub file_size_in_bytes: i64, -} - -impl PartitionStatisticsFile { - pub fn new( - snapshot_id: i64, - statistics_path: String, - file_size_in_bytes: i64, - ) -> PartitionStatisticsFile { - PartitionStatisticsFile { - snapshot_id, - statistics_path, - file_size_in_bytes, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/plan_status.rs b/crates/sail-catalog-iceberg/src/models/plan_status.rs deleted file mode 100644 index a23f1b3ad..000000000 --- a/crates/sail-catalog-iceberg/src/models/plan_status.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// PlanStatus : Status of a server-side planning operation -/// Status of a server-side planning operation -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum PlanStatus { - #[serde(rename = "completed")] - Completed, - #[serde(rename = "submitted")] - Submitted, - #[serde(rename = "cancelled")] - Cancelled, - #[serde(rename = "failed")] - Failed, -} - -impl std::fmt::Display for PlanStatus { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Completed => write!(f, "completed"), - Self::Submitted => write!(f, "submitted"), - Self::Cancelled => write!(f, "cancelled"), - Self::Failed => write!(f, "failed"), - } - } -} - -impl Default for PlanStatus { - fn default() -> PlanStatus { - Self::Completed - } -} diff --git a/crates/sail-catalog-iceberg/src/models/plan_table_scan_request.rs b/crates/sail-catalog-iceberg/src/models/plan_table_scan_request.rs deleted file mode 100644 index e92d932bc..000000000 --- a/crates/sail-catalog-iceberg/src/models/plan_table_scan_request.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct PlanTableScanRequest { - /// Identifier for the snapshot to scan in a point-in-time scan - #[serde(rename = "snapshot-id", skip_serializing_if = "Option::is_none")] - pub snapshot_id: Option, - /// List of selected schema fields - #[serde(rename = "select", skip_serializing_if = "Option::is_none")] - pub select: Option>, - /// Expression used to filter the table data - #[serde(rename = "filter", skip_serializing_if = "Option::is_none")] - pub filter: Option>, - /// Enables case sensitive field matching for filter and select - #[serde(rename = "case-sensitive", skip_serializing_if = "Option::is_none")] - pub case_sensitive: Option, - /// Whether to use the schema at the time the snapshot was written. When time travelling, the snapshot schema should be used (true). When scanning a branch, the table schema should be used (false). - #[serde( - rename = "use-snapshot-schema", - skip_serializing_if = "Option::is_none" - )] - pub use_snapshot_schema: Option, - /// Starting snapshot ID for an incremental scan (exclusive) - #[serde(rename = "start-snapshot-id", skip_serializing_if = "Option::is_none")] - pub start_snapshot_id: Option, - /// Ending snapshot ID for an incremental scan (inclusive). Required when start-snapshot-id is specified. - #[serde(rename = "end-snapshot-id", skip_serializing_if = "Option::is_none")] - pub end_snapshot_id: Option, - /// List of fields for which the service should send column stats. - #[serde(rename = "stats-fields", skip_serializing_if = "Option::is_none")] - pub stats_fields: Option>, -} - -impl PlanTableScanRequest { - pub fn new() -> PlanTableScanRequest { - PlanTableScanRequest { - snapshot_id: None, - select: None, - filter: None, - case_sensitive: None, - use_snapshot_schema: None, - start_snapshot_id: None, - end_snapshot_id: None, - stats_fields: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/plan_table_scan_result.rs b/crates/sail-catalog-iceberg/src/models/plan_table_scan_result.rs deleted file mode 100644 index 82c694cbe..000000000 --- a/crates/sail-catalog-iceberg/src/models/plan_table_scan_result.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// PlanTableScanResult : Result of server-side scan planning for planTableScan -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "status")] -pub enum PlanTableScanResult { - #[serde(rename = "completed")] - Completed(Box), - #[serde(rename = "failed")] - Failed(Box), - #[serde(rename = "submitted")] - Submitted(Box), - #[serde(rename = "cancelled")] - Cancelled(Box), -} - -impl Default for PlanTableScanResult { - fn default() -> Self { - Self::Completed(Default::default()) - } -} diff --git a/crates/sail-catalog-iceberg/src/models/position_delete_file.rs b/crates/sail-catalog-iceberg/src/models/position_delete_file.rs deleted file mode 100644 index f900817b8..000000000 --- a/crates/sail-catalog-iceberg/src/models/position_delete_file.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct PositionDeleteFile { - #[serde(rename = "content")] - pub content: String, - /// Offset within the delete file of delete content - #[serde(rename = "content-offset", skip_serializing_if = "Option::is_none")] - pub content_offset: Option, - /// Length, in bytes, of the delete content; required if content-offset is present - #[serde( - rename = "content-size-in-bytes", - skip_serializing_if = "Option::is_none" - )] - pub content_size_in_bytes: Option, - #[serde(rename = "file-path")] - pub file_path: String, - #[serde(rename = "file-format")] - pub file_format: models::FileFormat, - #[serde(rename = "spec-id")] - pub spec_id: i32, - /// A list of partition field values ordered based on the fields of the partition spec specified by the `spec-id` - #[serde(rename = "partition")] - pub partition: Vec, - /// Total file size in bytes - #[serde(rename = "file-size-in-bytes")] - pub file_size_in_bytes: i64, - /// Number of records in the file - #[serde(rename = "record-count")] - pub record_count: i64, - /// Encryption key metadata blob - #[serde(rename = "key-metadata", skip_serializing_if = "Option::is_none")] - pub key_metadata: Option, - /// List of splittable offsets - #[serde(rename = "split-offsets", skip_serializing_if = "Option::is_none")] - pub split_offsets: Option>, - #[serde(rename = "sort-order-id", skip_serializing_if = "Option::is_none")] - pub sort_order_id: Option, -} - -impl PositionDeleteFile { - pub fn new( - content: String, - file_path: String, - file_format: models::FileFormat, - spec_id: i32, - partition: Vec, - file_size_in_bytes: i64, - record_count: i64, - ) -> PositionDeleteFile { - PositionDeleteFile { - content, - content_offset: None, - content_size_in_bytes: None, - file_path, - file_format, - spec_id, - partition, - file_size_in_bytes, - record_count, - key_metadata: None, - split_offsets: None, - sort_order_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/primitive_type_value.rs b/crates/sail-catalog-iceberg/src/models/primitive_type_value.rs deleted file mode 100644 index 76356e519..000000000 --- a/crates/sail-catalog-iceberg/src/models/primitive_type_value.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum PrimitiveTypeValue { - BooleanTypeValue(bool), - IntegerTypeValue(i32), - LongTypeValue(i64), - FloatTypeValue(f32), - DoubleTypeValue(f64), - /// Decimal type values are serialized as strings. Decimals with a positive scale serialize as numeric plain text, while decimals with a negative scale use scientific notation and the exponent will be equal to the negated scale. For instance, a decimal with a positive scale is '123.4500', with zero scale is '2', and with a negative scale is '2E+20' - DecimalTypeValue(String), - StringTypeValue(String), - /// UUID type values are serialized as a 36-character lowercase string in standard UUID format as specified by RFC-4122 - UuidTypeValue(uuid::Uuid), - /// Date type values follow the 'YYYY-MM-DD' ISO-8601 standard date format - DateTypeValue(String), - /// Time type values follow the 'HH:MM:SS.ssssss' ISO-8601 format with microsecond precision - TimeTypeValue(String), - /// Timestamp type values follow the 'YYYY-MM-DDTHH:MM:SS.ssssss' ISO-8601 format with microsecond precision - TimestampTypeValue(String), - /// TimestampTz type values follow the 'YYYY-MM-DDTHH:MM:SS.ssssss+00:00' ISO-8601 format with microsecond precision, and a timezone offset (+00:00 for UTC) - TimestampTzTypeValue(String), - /// Timestamp_ns type values follow the 'YYYY-MM-DDTHH:MM:SS.sssssssss' ISO-8601 format with nanosecond precision - TimestampNanoTypeValue(String), - /// Timestamp_ns type values follow the 'YYYY-MM-DDTHH:MM:SS.sssssssss+00:00' ISO-8601 format with nanosecond precision, and a timezone offset (+00:00 for UTC) - TimestampTzNanoTypeValue(String), - /// Fixed length type values are stored and serialized as an uppercase hexadecimal string preserving the fixed length - FixedTypeValue(String), - /// Binary type values are stored and serialized as an uppercase hexadecimal string - BinaryTypeValue(String), -} - -impl Default for PrimitiveTypeValue { - fn default() -> Self { - Self::BooleanTypeValue(Default::default()) - } -} diff --git a/crates/sail-catalog-iceberg/src/models/register_table_request.rs b/crates/sail-catalog-iceberg/src/models/register_table_request.rs deleted file mode 100644 index 215f0a2b8..000000000 --- a/crates/sail-catalog-iceberg/src/models/register_table_request.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RegisterTableRequest { - #[serde(rename = "name")] - pub name: String, - #[serde(rename = "metadata-location")] - pub metadata_location: String, - /// Whether to overwrite table metadata if the table already exists - #[serde(rename = "overwrite", skip_serializing_if = "Option::is_none")] - pub overwrite: Option, -} - -impl RegisterTableRequest { - pub fn new(name: String, metadata_location: String) -> RegisterTableRequest { - RegisterTableRequest { - name, - metadata_location, - overwrite: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_encryption_key_update.rs b/crates/sail-catalog-iceberg/src/models/remove_encryption_key_update.rs deleted file mode 100644 index 1afd05835..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_encryption_key_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemoveEncryptionKeyUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "key-id")] - pub key_id: String, -} - -impl RemoveEncryptionKeyUpdate { - pub fn new(key_id: String) -> RemoveEncryptionKeyUpdate { - RemoveEncryptionKeyUpdate { - action: None, - key_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_partition_specs_update.rs b/crates/sail-catalog-iceberg/src/models/remove_partition_specs_update.rs deleted file mode 100644 index a891f457d..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_partition_specs_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemovePartitionSpecsUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "spec-ids")] - pub spec_ids: Vec, -} - -impl RemovePartitionSpecsUpdate { - pub fn new(spec_ids: Vec) -> RemovePartitionSpecsUpdate { - RemovePartitionSpecsUpdate { - action: None, - spec_ids, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_partition_statistics_update.rs b/crates/sail-catalog-iceberg/src/models/remove_partition_statistics_update.rs deleted file mode 100644 index 78d4d8a37..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_partition_statistics_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemovePartitionStatisticsUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, -} - -impl RemovePartitionStatisticsUpdate { - pub fn new(snapshot_id: i64) -> RemovePartitionStatisticsUpdate { - RemovePartitionStatisticsUpdate { - action: None, - snapshot_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_properties_update.rs b/crates/sail-catalog-iceberg/src/models/remove_properties_update.rs deleted file mode 100644 index 30ec0c27c..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_properties_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemovePropertiesUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "removals")] - pub removals: Vec, -} - -impl RemovePropertiesUpdate { - pub fn new(removals: Vec) -> RemovePropertiesUpdate { - RemovePropertiesUpdate { - action: None, - removals, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_schemas_update.rs b/crates/sail-catalog-iceberg/src/models/remove_schemas_update.rs deleted file mode 100644 index 5c2d1bc38..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_schemas_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemoveSchemasUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "schema-ids")] - pub schema_ids: Vec, -} - -impl RemoveSchemasUpdate { - pub fn new(schema_ids: Vec) -> RemoveSchemasUpdate { - RemoveSchemasUpdate { - action: None, - schema_ids, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_snapshot_ref_update.rs b/crates/sail-catalog-iceberg/src/models/remove_snapshot_ref_update.rs deleted file mode 100644 index e02e265ab..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_snapshot_ref_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemoveSnapshotRefUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "ref-name")] - pub ref_name: String, -} - -impl RemoveSnapshotRefUpdate { - pub fn new(ref_name: String) -> RemoveSnapshotRefUpdate { - RemoveSnapshotRefUpdate { - action: None, - ref_name, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_snapshots_update.rs b/crates/sail-catalog-iceberg/src/models/remove_snapshots_update.rs deleted file mode 100644 index 8249cccf7..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_snapshots_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemoveSnapshotsUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "snapshot-ids")] - pub snapshot_ids: Vec, -} - -impl RemoveSnapshotsUpdate { - pub fn new(snapshot_ids: Vec) -> RemoveSnapshotsUpdate { - RemoveSnapshotsUpdate { - action: None, - snapshot_ids, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/remove_statistics_update.rs b/crates/sail-catalog-iceberg/src/models/remove_statistics_update.rs deleted file mode 100644 index 30760a461..000000000 --- a/crates/sail-catalog-iceberg/src/models/remove_statistics_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RemoveStatisticsUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, -} - -impl RemoveStatisticsUpdate { - pub fn new(snapshot_id: i64) -> RemoveStatisticsUpdate { - RemoveStatisticsUpdate { - action: None, - snapshot_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/rename_table_request.rs b/crates/sail-catalog-iceberg/src/models/rename_table_request.rs deleted file mode 100644 index c82c5f208..000000000 --- a/crates/sail-catalog-iceberg/src/models/rename_table_request.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct RenameTableRequest { - #[serde(rename = "source")] - pub source: Box, - #[serde(rename = "destination")] - pub destination: Box, -} - -impl RenameTableRequest { - pub fn new( - source: models::TableIdentifier, - destination: models::TableIdentifier, - ) -> RenameTableRequest { - RenameTableRequest { - source: Box::new(source), - destination: Box::new(destination), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/report_metrics_request.rs b/crates/sail-catalog-iceberg/src/models/report_metrics_request.rs deleted file mode 100644 index e9799701b..000000000 --- a/crates/sail-catalog-iceberg/src/models/report_metrics_request.rs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ReportMetricsRequest { - #[serde(rename = "report-type")] - pub report_type: String, - #[serde(rename = "table-name")] - pub table_name: String, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "filter")] - pub filter: Box, - #[serde(rename = "schema-id")] - pub schema_id: i32, - #[serde(rename = "projected-field-ids")] - pub projected_field_ids: Vec, - #[serde(rename = "projected-field-names")] - pub projected_field_names: Vec, - #[serde(rename = "metrics")] - pub metrics: std::collections::HashMap, - #[serde(rename = "metadata", skip_serializing_if = "Option::is_none")] - pub metadata: Option>, - #[serde(rename = "sequence-number")] - pub sequence_number: i64, - #[serde(rename = "operation")] - pub operation: String, -} - -impl ReportMetricsRequest { - pub fn new( - report_type: String, - table_name: String, - snapshot_id: i64, - filter: models::Expression, - schema_id: i32, - projected_field_ids: Vec, - projected_field_names: Vec, - metrics: std::collections::HashMap, - sequence_number: i64, - operation: String, - ) -> ReportMetricsRequest { - ReportMetricsRequest { - report_type, - table_name, - snapshot_id, - filter: Box::new(filter), - schema_id, - projected_field_ids, - projected_field_names, - metrics, - metadata: None, - sequence_number, - operation, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/scan_report.rs b/crates/sail-catalog-iceberg/src/models/scan_report.rs deleted file mode 100644 index b00648374..000000000 --- a/crates/sail-catalog-iceberg/src/models/scan_report.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ScanReport { - #[serde(rename = "table-name")] - pub table_name: String, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "filter")] - pub filter: Box, - #[serde(rename = "schema-id")] - pub schema_id: i32, - #[serde(rename = "projected-field-ids")] - pub projected_field_ids: Vec, - #[serde(rename = "projected-field-names")] - pub projected_field_names: Vec, - #[serde(rename = "metrics")] - pub metrics: std::collections::HashMap, - #[serde(rename = "metadata", skip_serializing_if = "Option::is_none")] - pub metadata: Option>, -} - -impl ScanReport { - pub fn new( - table_name: String, - snapshot_id: i64, - filter: models::Expression, - schema_id: i32, - projected_field_ids: Vec, - projected_field_names: Vec, - metrics: std::collections::HashMap, - ) -> ScanReport { - ScanReport { - table_name, - snapshot_id, - filter: Box::new(filter), - schema_id, - projected_field_ids, - projected_field_names, - metrics, - metadata: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/scan_tasks.rs b/crates/sail-catalog-iceberg/src/models/scan_tasks.rs deleted file mode 100644 index dce5baa95..000000000 --- a/crates/sail-catalog-iceberg/src/models/scan_tasks.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// ScanTasks : Scan and planning tasks for server-side scan planning - `plan-tasks` contains opaque units of planning work - `file-scan-tasks` contains a partial or complete list of table scan tasks - `delete-files` contains delete files referenced by file scan tasks Each plan task must be passed to the fetchScanTasks endpoint to fetch the file scan tasks for the plan task. The list of delete files must contain all delete files referenced by the file scan tasks. -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ScanTasks { - /// Delete files referenced by file scan tasks - #[serde(rename = "delete-files", skip_serializing_if = "Option::is_none")] - pub delete_files: Option>, - #[serde(rename = "file-scan-tasks", skip_serializing_if = "Option::is_none")] - pub file_scan_tasks: Option>, - #[serde(rename = "plan-tasks", skip_serializing_if = "Option::is_none")] - pub plan_tasks: Option>, -} - -impl ScanTasks { - /// Scan and planning tasks for server-side scan planning - `plan-tasks` contains opaque units of planning work - `file-scan-tasks` contains a partial or complete list of table scan tasks - `delete-files` contains delete files referenced by file scan tasks Each plan task must be passed to the fetchScanTasks endpoint to fetch the file scan tasks for the plan task. The list of delete files must contain all delete files referenced by the file scan tasks. - pub fn new() -> ScanTasks { - ScanTasks { - delete_files: None, - file_scan_tasks: None, - plan_tasks: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/schema.rs b/crates/sail-catalog-iceberg/src/models/schema.rs deleted file mode 100644 index ac133918e..000000000 --- a/crates/sail-catalog-iceberg/src/models/schema.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use sail_iceberg::NestedFieldRef; -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// Ref: https://iceberg.apache.org/javadoc/latest/org/apache/iceberg/Schema.html -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct Schema { - #[serde(rename = "type")] - pub r#type: Type, - #[serde(rename = "fields")] - pub fields: Vec, - #[serde(rename = "schema-id", skip_serializing_if = "Option::is_none")] - pub schema_id: Option, - // Identifier is a concept similar to primary key in a relational database system. - // It consists of a unique set of primitive fields in the schema. - // An identifier field must be at root, or nested in a chain of structs (no maps or lists). - // A row should be unique in a table based on the values of the identifier fields. - // Optional, float and double columns cannot be used as identifier fields. - // However, Iceberg identifier differs from primary key in the following ways: - // - Iceberg does not enforce the uniqueness of a row based on this identifier information. - // It is used for operations like upsert to define the default upsert key. - // - A nested field in a struct can be used as an identifier. - // For example, if there is a "last_name" field inside a "user" struct in a schema, - // field "user.last_name" can be set as a part of the identifier field. - #[serde( - rename = "identifier-field-ids", - skip_serializing_if = "Option::is_none" - )] - pub identifier_field_ids: Option>, -} - -impl Schema { - pub fn new(r#type: Type, fields: Vec) -> Schema { - Schema { - r#type, - fields, - schema_id: None, - identifier_field_ids: None, - } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Type { - #[serde(rename = "struct")] - Struct, -} - -impl Default for Type { - fn default() -> Type { - Self::Struct - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_current_schema_update.rs b/crates/sail-catalog-iceberg/src/models/set_current_schema_update.rs deleted file mode 100644 index ad744871d..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_current_schema_update.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetCurrentSchemaUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - /// Schema ID to set as current, or -1 to set last added schema - #[serde(rename = "schema-id")] - pub schema_id: i32, -} - -impl SetCurrentSchemaUpdate { - pub fn new(schema_id: i32) -> SetCurrentSchemaUpdate { - SetCurrentSchemaUpdate { - action: None, - schema_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_current_view_version_update.rs b/crates/sail-catalog-iceberg/src/models/set_current_view_version_update.rs deleted file mode 100644 index c3189c1a8..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_current_view_version_update.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetCurrentViewVersionUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - /// The view version id to set as current, or -1 to set last added view version id - #[serde(rename = "view-version-id")] - pub view_version_id: i32, -} - -impl SetCurrentViewVersionUpdate { - pub fn new(view_version_id: i32) -> SetCurrentViewVersionUpdate { - SetCurrentViewVersionUpdate { - action: None, - view_version_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_default_sort_order_update.rs b/crates/sail-catalog-iceberg/src/models/set_default_sort_order_update.rs deleted file mode 100644 index 8969c1ebe..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_default_sort_order_update.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetDefaultSortOrderUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - /// Sort order ID to set as the default, or -1 to set last added sort order - #[serde(rename = "sort-order-id")] - pub sort_order_id: i32, -} - -impl SetDefaultSortOrderUpdate { - pub fn new(sort_order_id: i32) -> SetDefaultSortOrderUpdate { - SetDefaultSortOrderUpdate { - action: None, - sort_order_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_default_spec_update.rs b/crates/sail-catalog-iceberg/src/models/set_default_spec_update.rs deleted file mode 100644 index 089be386b..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_default_spec_update.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetDefaultSpecUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - /// Partition spec ID to set as the default, or -1 to set last added spec - #[serde(rename = "spec-id")] - pub spec_id: i32, -} - -impl SetDefaultSpecUpdate { - pub fn new(spec_id: i32) -> SetDefaultSpecUpdate { - SetDefaultSpecUpdate { - action: None, - spec_id, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_expression.rs b/crates/sail-catalog-iceberg/src/models/set_expression.rs deleted file mode 100644 index af3653561..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_expression.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetExpression { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "term")] - pub term: Box, - #[serde(rename = "values")] - pub values: Vec, -} - -impl SetExpression { - pub fn new( - r#type: String, - term: models::Term, - values: Vec, - ) -> SetExpression { - SetExpression { - r#type, - term: Box::new(term), - values, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_location_update.rs b/crates/sail-catalog-iceberg/src/models/set_location_update.rs deleted file mode 100644 index bc12907d7..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_location_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetLocationUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "location")] - pub location: String, -} - -impl SetLocationUpdate { - pub fn new(location: String) -> SetLocationUpdate { - SetLocationUpdate { - action: None, - location, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_partition_statistics_update.rs b/crates/sail-catalog-iceberg/src/models/set_partition_statistics_update.rs deleted file mode 100644 index 801a1761c..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_partition_statistics_update.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetPartitionStatisticsUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "partition-statistics")] - pub partition_statistics: Box, -} - -impl SetPartitionStatisticsUpdate { - pub fn new( - partition_statistics: models::PartitionStatisticsFile, - ) -> SetPartitionStatisticsUpdate { - SetPartitionStatisticsUpdate { - action: None, - partition_statistics: Box::new(partition_statistics), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_properties_update.rs b/crates/sail-catalog-iceberg/src/models/set_properties_update.rs deleted file mode 100644 index 7e0d8fd1f..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_properties_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetPropertiesUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "updates")] - pub updates: std::collections::HashMap, -} - -impl SetPropertiesUpdate { - pub fn new(updates: std::collections::HashMap) -> SetPropertiesUpdate { - SetPropertiesUpdate { - action: None, - updates, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_snapshot_ref_update.rs b/crates/sail-catalog-iceberg/src/models/set_snapshot_ref_update.rs deleted file mode 100644 index e637b4cfe..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_snapshot_ref_update.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetSnapshotRefUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "ref-name")] - pub ref_name: String, - #[serde(rename = "type")] - pub r#type: Type, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "max-ref-age-ms", skip_serializing_if = "Option::is_none")] - pub max_ref_age_ms: Option, - #[serde( - rename = "max-snapshot-age-ms", - skip_serializing_if = "Option::is_none" - )] - pub max_snapshot_age_ms: Option, - #[serde( - rename = "min-snapshots-to-keep", - skip_serializing_if = "Option::is_none" - )] - pub min_snapshots_to_keep: Option, -} - -impl SetSnapshotRefUpdate { - pub fn new(ref_name: String, r#type: Type, snapshot_id: i64) -> SetSnapshotRefUpdate { - SetSnapshotRefUpdate { - action: None, - ref_name, - r#type, - snapshot_id, - max_ref_age_ms: None, - max_snapshot_age_ms: None, - min_snapshots_to_keep: None, - } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Type { - #[serde(rename = "tag")] - Tag, - #[serde(rename = "branch")] - Branch, -} - -impl Default for Type { - fn default() -> Type { - Self::Tag - } -} diff --git a/crates/sail-catalog-iceberg/src/models/set_statistics_update.rs b/crates/sail-catalog-iceberg/src/models/set_statistics_update.rs deleted file mode 100644 index cca9e89a9..000000000 --- a/crates/sail-catalog-iceberg/src/models/set_statistics_update.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SetStatisticsUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - /// This optional field is **DEPRECATED for REMOVAL** since it contains redundant information. Clients should use the `statistics.snapshot-id` field instead. - #[serde(rename = "snapshot-id", skip_serializing_if = "Option::is_none")] - pub snapshot_id: Option, - #[serde(rename = "statistics")] - pub statistics: Box, -} - -impl SetStatisticsUpdate { - pub fn new(statistics: models::StatisticsFile) -> SetStatisticsUpdate { - SetStatisticsUpdate { - action: None, - snapshot_id: None, - statistics: Box::new(statistics), - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/snapshot.rs b/crates/sail-catalog-iceberg/src/models/snapshot.rs deleted file mode 100644 index b1fd3e2eb..000000000 --- a/crates/sail-catalog-iceberg/src/models/snapshot.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct Snapshot { - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "parent-snapshot-id", skip_serializing_if = "Option::is_none")] - pub parent_snapshot_id: Option, - #[serde(rename = "sequence-number", skip_serializing_if = "Option::is_none")] - pub sequence_number: Option, - #[serde(rename = "timestamp-ms")] - pub timestamp_ms: i64, - /// Location of the snapshot's manifest list file - #[serde(rename = "manifest-list")] - pub manifest_list: String, - /// The first _row_id assigned to the first row in the first data file in the first manifest - #[serde(rename = "first-row-id", skip_serializing_if = "Option::is_none")] - pub first_row_id: Option, - /// The upper bound of the number of rows with assigned row IDs - #[serde(rename = "added-rows", skip_serializing_if = "Option::is_none")] - pub added_rows: Option, - #[serde(rename = "summary")] - pub summary: models::SnapshotSummary, - #[serde(rename = "schema-id", skip_serializing_if = "Option::is_none")] - pub schema_id: Option, -} - -impl Snapshot { - pub fn new( - snapshot_id: i64, - timestamp_ms: i64, - manifest_list: String, - summary: models::SnapshotSummary, - ) -> Snapshot { - Snapshot { - snapshot_id, - parent_snapshot_id: None, - sequence_number: None, - timestamp_ms, - manifest_list, - first_row_id: None, - added_rows: None, - summary, - schema_id: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/snapshot_log_inner.rs b/crates/sail-catalog-iceberg/src/models/snapshot_log_inner.rs deleted file mode 100644 index 6cabc8cd0..000000000 --- a/crates/sail-catalog-iceberg/src/models/snapshot_log_inner.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SnapshotLogInner { - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "timestamp-ms")] - pub timestamp_ms: i64, -} - -impl SnapshotLogInner { - pub fn new(snapshot_id: i64, timestamp_ms: i64) -> SnapshotLogInner { - SnapshotLogInner { - snapshot_id, - timestamp_ms, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/snapshot_reference.rs b/crates/sail-catalog-iceberg/src/models/snapshot_reference.rs deleted file mode 100644 index 8ce0c1aff..000000000 --- a/crates/sail-catalog-iceberg/src/models/snapshot_reference.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SnapshotReference { - #[serde(rename = "type")] - pub r#type: Type, - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "max-ref-age-ms", skip_serializing_if = "Option::is_none")] - pub max_ref_age_ms: Option, - #[serde( - rename = "max-snapshot-age-ms", - skip_serializing_if = "Option::is_none" - )] - pub max_snapshot_age_ms: Option, - #[serde( - rename = "min-snapshots-to-keep", - skip_serializing_if = "Option::is_none" - )] - pub min_snapshots_to_keep: Option, -} - -impl SnapshotReference { - pub fn new(r#type: Type, snapshot_id: i64) -> SnapshotReference { - SnapshotReference { - r#type, - snapshot_id, - max_ref_age_ms: None, - max_snapshot_age_ms: None, - min_snapshots_to_keep: None, - } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Type { - #[serde(rename = "tag")] - Tag, - #[serde(rename = "branch")] - Branch, -} - -impl Default for Type { - fn default() -> Type { - Self::Tag - } -} diff --git a/crates/sail-catalog-iceberg/src/models/snapshot_summary.rs b/crates/sail-catalog-iceberg/src/models/snapshot_summary.rs deleted file mode 100644 index ca82dbba3..000000000 --- a/crates/sail-catalog-iceberg/src/models/snapshot_summary.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SnapshotSummary { - #[serde(rename = "operation")] - pub operation: Operation, -} - -impl SnapshotSummary { - pub fn new(operation: Operation) -> SnapshotSummary { - SnapshotSummary { operation } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Operation { - #[serde(rename = "append")] - Append, - #[serde(rename = "replace")] - Replace, - #[serde(rename = "overwrite")] - Overwrite, - #[serde(rename = "delete")] - Delete, -} - -impl Default for Operation { - fn default() -> Operation { - Self::Append - } -} diff --git a/crates/sail-catalog-iceberg/src/models/sort_direction.rs b/crates/sail-catalog-iceberg/src/models/sort_direction.rs deleted file mode 100644 index ae0a0ad8c..000000000 --- a/crates/sail-catalog-iceberg/src/models/sort_direction.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum SortDirection { - #[serde(rename = "asc")] - Asc, - #[serde(rename = "desc")] - Desc, -} - -impl std::fmt::Display for SortDirection { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Asc => write!(f, "asc"), - Self::Desc => write!(f, "desc"), - } - } -} - -impl Default for SortDirection { - fn default() -> SortDirection { - Self::Asc - } -} diff --git a/crates/sail-catalog-iceberg/src/models/sort_field.rs b/crates/sail-catalog-iceberg/src/models/sort_field.rs deleted file mode 100644 index 9fd2ad5ef..000000000 --- a/crates/sail-catalog-iceberg/src/models/sort_field.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SortField { - #[serde(rename = "source-id")] - pub source_id: i32, - #[serde(rename = "transform")] - pub transform: String, - #[serde(rename = "direction")] - pub direction: models::SortDirection, - #[serde(rename = "null-order")] - pub null_order: models::NullOrder, -} - -impl SortField { - pub fn new( - source_id: i32, - transform: String, - direction: models::SortDirection, - null_order: models::NullOrder, - ) -> SortField { - SortField { - source_id, - transform, - direction, - null_order, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/sort_order.rs b/crates/sail-catalog-iceberg/src/models/sort_order.rs deleted file mode 100644 index b70deb445..000000000 --- a/crates/sail-catalog-iceberg/src/models/sort_order.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SortOrder { - #[serde(rename = "order-id")] - pub order_id: i32, - #[serde(rename = "fields")] - pub fields: Vec, -} - -impl SortOrder { - pub fn new(order_id: i32, fields: Vec) -> SortOrder { - SortOrder { order_id, fields } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/sql_view_representation.rs b/crates/sail-catalog-iceberg/src/models/sql_view_representation.rs deleted file mode 100644 index d33d65fe9..000000000 --- a/crates/sail-catalog-iceberg/src/models/sql_view_representation.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct SqlViewRepresentation { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "sql")] - pub sql: String, - #[serde(rename = "dialect")] - pub dialect: String, -} - -impl SqlViewRepresentation { - pub fn new(r#type: String, sql: String, dialect: String) -> SqlViewRepresentation { - SqlViewRepresentation { - r#type, - sql, - dialect, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/statistics_file.rs b/crates/sail-catalog-iceberg/src/models/statistics_file.rs deleted file mode 100644 index 8c5bada63..000000000 --- a/crates/sail-catalog-iceberg/src/models/statistics_file.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct StatisticsFile { - #[serde(rename = "snapshot-id")] - pub snapshot_id: i64, - #[serde(rename = "statistics-path")] - pub statistics_path: String, - #[serde(rename = "file-size-in-bytes")] - pub file_size_in_bytes: i64, - #[serde(rename = "file-footer-size-in-bytes")] - pub file_footer_size_in_bytes: i64, - #[serde(rename = "blob-metadata")] - pub blob_metadata: Vec, -} - -impl StatisticsFile { - pub fn new( - snapshot_id: i64, - statistics_path: String, - file_size_in_bytes: i64, - file_footer_size_in_bytes: i64, - blob_metadata: Vec, - ) -> StatisticsFile { - StatisticsFile { - snapshot_id, - statistics_path, - file_size_in_bytes, - file_footer_size_in_bytes, - blob_metadata, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/storage_credential.rs b/crates/sail-catalog-iceberg/src/models/storage_credential.rs deleted file mode 100644 index 091d33411..000000000 --- a/crates/sail-catalog-iceberg/src/models/storage_credential.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct StorageCredential { - /// Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix (by selecting the longest prefix) if several credentials of the same type are available. - #[serde(rename = "prefix")] - pub prefix: String, - #[serde(rename = "config")] - pub config: std::collections::HashMap, -} - -impl StorageCredential { - pub fn new( - prefix: String, - config: std::collections::HashMap, - ) -> StorageCredential { - StorageCredential { prefix, config } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/table_identifier.rs b/crates/sail-catalog-iceberg/src/models/table_identifier.rs deleted file mode 100644 index 3f1407785..000000000 --- a/crates/sail-catalog-iceberg/src/models/table_identifier.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct TableIdentifier { - /// Reference to one or more levels of a namespace - #[serde(rename = "namespace")] - pub namespace: Vec, - #[serde(rename = "name")] - pub name: String, -} - -impl TableIdentifier { - pub fn new(namespace: Vec, name: String) -> TableIdentifier { - TableIdentifier { namespace, name } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/table_metadata.rs b/crates/sail-catalog-iceberg/src/models/table_metadata.rs deleted file mode 100644 index e4ac29a31..000000000 --- a/crates/sail-catalog-iceberg/src/models/table_metadata.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct TableMetadata { - #[serde(rename = "format-version")] - pub format_version: i32, - #[serde(rename = "table-uuid")] - pub table_uuid: String, - #[serde(rename = "location", skip_serializing_if = "Option::is_none")] - pub location: Option, - #[serde(rename = "last-updated-ms", skip_serializing_if = "Option::is_none")] - pub last_updated_ms: Option, - /// A long higher than all assigned row IDs; the next snapshot's first-row-id. - #[serde(rename = "next-row-id", skip_serializing_if = "Option::is_none")] - pub next_row_id: Option, - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, - #[serde(rename = "schemas", skip_serializing_if = "Option::is_none")] - pub schemas: Option>, - #[serde(rename = "current-schema-id", skip_serializing_if = "Option::is_none")] - pub current_schema_id: Option, - #[serde(rename = "last-column-id", skip_serializing_if = "Option::is_none")] - pub last_column_id: Option, - #[serde(rename = "partition-specs", skip_serializing_if = "Option::is_none")] - pub partition_specs: Option>, - #[serde(rename = "default-spec-id", skip_serializing_if = "Option::is_none")] - pub default_spec_id: Option, - #[serde(rename = "last-partition-id", skip_serializing_if = "Option::is_none")] - pub last_partition_id: Option, - #[serde(rename = "sort-orders", skip_serializing_if = "Option::is_none")] - pub sort_orders: Option>, - #[serde( - rename = "default-sort-order-id", - skip_serializing_if = "Option::is_none" - )] - pub default_sort_order_id: Option, - #[serde(rename = "encryption-keys", skip_serializing_if = "Option::is_none")] - pub encryption_keys: Option>, - #[serde(rename = "snapshots", skip_serializing_if = "Option::is_none")] - pub snapshots: Option>, - #[serde(rename = "refs", skip_serializing_if = "Option::is_none")] - pub refs: Option>, - #[serde( - rename = "current-snapshot-id", - skip_serializing_if = "Option::is_none" - )] - pub current_snapshot_id: Option, - #[serde( - rename = "last-sequence-number", - skip_serializing_if = "Option::is_none" - )] - pub last_sequence_number: Option, - #[serde(rename = "snapshot-log", skip_serializing_if = "Option::is_none")] - pub snapshot_log: Option>, - #[serde(rename = "metadata-log", skip_serializing_if = "Option::is_none")] - pub metadata_log: Option>, - #[serde(rename = "statistics", skip_serializing_if = "Option::is_none")] - pub statistics: Option>, - #[serde( - rename = "partition-statistics", - skip_serializing_if = "Option::is_none" - )] - pub partition_statistics: Option>, -} - -impl TableMetadata { - pub fn new(format_version: i32, table_uuid: String) -> TableMetadata { - TableMetadata { - format_version, - table_uuid, - location: None, - last_updated_ms: None, - next_row_id: None, - properties: None, - schemas: None, - current_schema_id: None, - last_column_id: None, - partition_specs: None, - default_spec_id: None, - last_partition_id: None, - sort_orders: None, - default_sort_order_id: None, - encryption_keys: None, - snapshots: None, - refs: None, - current_snapshot_id: None, - last_sequence_number: None, - snapshot_log: None, - metadata_log: None, - statistics: None, - partition_statistics: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/table_requirement.rs b/crates/sail-catalog-iceberg/src/models/table_requirement.rs deleted file mode 100644 index 2b4714fe4..000000000 --- a/crates/sail-catalog-iceberg/src/models/table_requirement.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum TableRequirement { - #[serde(rename = "assert-create")] - AssertCreate {}, - #[serde(rename = "assert-current-schema-id")] - AssertCurrentSchemaId {}, - #[serde(rename = "assert-default-sort-order-id")] - AssertDefaultSortOrderId {}, - #[serde(rename = "assert-default-spec-id")] - AssertDefaultSpecId {}, - #[serde(rename = "assert-last-assigned-field-id")] - AssertLastAssignedFieldId {}, - #[serde(rename = "assert-last-assigned-partition-id")] - AssertLastAssignedPartitionId {}, - #[serde(rename = "assert-ref-snapshot-id")] - AssertRefSnapshotId {}, - #[serde(rename = "assert-table-uuid")] - AssertTableUuid {}, -} - -impl Default for TableRequirement { - fn default() -> Self { - Self::AssertCreate {} - } -} diff --git a/crates/sail-catalog-iceberg/src/models/table_update.rs b/crates/sail-catalog-iceberg/src/models/table_update.rs deleted file mode 100644 index 52dd12a88..000000000 --- a/crates/sail-catalog-iceberg/src/models/table_update.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "action")] -pub enum TableUpdate { - #[serde(rename = "add-encryption-key")] - AddEncryptionKeyUpdate {}, - #[serde(rename = "add-schema")] - AddSchemaUpdate {}, - #[serde(rename = "add-snapshot")] - AddSnapshotUpdate {}, - #[serde(rename = "add-sort-order")] - AddSortOrderUpdate {}, - #[serde(rename = "add-spec")] - AddPartitionSpecUpdate {}, - #[serde(rename = "assign-uuid")] - AssignUuidUpdate {}, - #[serde(rename = "remove-encryption-key")] - RemoveEncryptionKeyUpdate {}, - #[serde(rename = "remove-partition-specs")] - RemovePartitionSpecsUpdate {}, - #[serde(rename = "remove-properties")] - RemovePropertiesUpdate {}, - #[serde(rename = "remove-schemas")] - RemoveSchemasUpdate {}, - #[serde(rename = "remove-snapshot-ref")] - RemoveSnapshotRefUpdate {}, - #[serde(rename = "remove-snapshots")] - RemoveSnapshotsUpdate {}, - #[serde(rename = "remove-statistics")] - RemoveStatisticsUpdate {}, - #[serde(rename = "set-current-schema")] - SetCurrentSchemaUpdate {}, - #[serde(rename = "set-default-sort-order")] - SetDefaultSortOrderUpdate {}, - #[serde(rename = "set-default-spec")] - SetDefaultSpecUpdate {}, - #[serde(rename = "set-location")] - SetLocationUpdate {}, - #[serde(rename = "set-properties")] - SetPropertiesUpdate {}, - #[serde(rename = "set-snapshot-ref")] - SetSnapshotRefUpdate {}, - #[serde(rename = "set-statistics")] - SetStatisticsUpdate {}, - #[serde(rename = "upgrade-format-version")] - UpgradeFormatVersionUpdate {}, -} - -impl Default for TableUpdate { - fn default() -> Self { - Self::AddEncryptionKeyUpdate {} - } -} diff --git a/crates/sail-catalog-iceberg/src/models/term.rs b/crates/sail-catalog-iceberg/src/models/term.rs deleted file mode 100644 index bd518c530..000000000 --- a/crates/sail-catalog-iceberg/src/models/term.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum Term { - Reference(String), - TransformTerm(Box), -} - -impl Default for Term { - fn default() -> Self { - Self::Reference(Default::default()) - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Type { - #[serde(rename = "transform")] - Transform, -} - -impl Default for Type { - fn default() -> Type { - Self::Transform - } -} diff --git a/crates/sail-catalog-iceberg/src/models/timer_result.rs b/crates/sail-catalog-iceberg/src/models/timer_result.rs deleted file mode 100644 index 82dbe33e7..000000000 --- a/crates/sail-catalog-iceberg/src/models/timer_result.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct TimerResult { - #[serde(rename = "time-unit")] - pub time_unit: String, - #[serde(rename = "count")] - pub count: i64, - #[serde(rename = "total-duration")] - pub total_duration: i64, -} - -impl TimerResult { - pub fn new(time_unit: String, count: i64, total_duration: i64) -> TimerResult { - TimerResult { - time_unit, - count, - total_duration, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/token_type.rs b/crates/sail-catalog-iceberg/src/models/token_type.rs deleted file mode 100644 index c1a2f4f49..000000000 --- a/crates/sail-catalog-iceberg/src/models/token_type.rs +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -/// TokenType : Token type identifier, from RFC 8693 Section 3 See https://datatracker.ietf.org/doc/html/rfc8693#section-3 -/// Token type identifier, from RFC 8693 Section 3 See https://datatracker.ietf.org/doc/html/rfc8693#section-3 -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum TokenType { - #[serde(rename = "urn:ietf:params:oauth:token-type:access_token")] - UrnColonIetfColonParamsColonOauthColonTokenTypeColonAccessToken, - #[serde(rename = "urn:ietf:params:oauth:token-type:refresh_token")] - UrnColonIetfColonParamsColonOauthColonTokenTypeColonRefreshToken, - #[serde(rename = "urn:ietf:params:oauth:token-type:id_token")] - UrnColonIetfColonParamsColonOauthColonTokenTypeColonIdToken, - #[serde(rename = "urn:ietf:params:oauth:token-type:saml1")] - UrnColonIetfColonParamsColonOauthColonTokenTypeColonSaml1, - #[serde(rename = "urn:ietf:params:oauth:token-type:saml2")] - UrnColonIetfColonParamsColonOauthColonTokenTypeColonSaml2, - #[serde(rename = "urn:ietf:params:oauth:token-type:jwt")] - UrnColonIetfColonParamsColonOauthColonTokenTypeColonJwt, -} - -impl std::fmt::Display for TokenType { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonAccessToken => { - write!(f, "urn:ietf:params:oauth:token-type:access_token") - } - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonRefreshToken => { - write!(f, "urn:ietf:params:oauth:token-type:refresh_token") - } - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonIdToken => { - write!(f, "urn:ietf:params:oauth:token-type:id_token") - } - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonSaml1 => { - write!(f, "urn:ietf:params:oauth:token-type:saml1") - } - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonSaml2 => { - write!(f, "urn:ietf:params:oauth:token-type:saml2") - } - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonJwt => { - write!(f, "urn:ietf:params:oauth:token-type:jwt") - } - } - } -} - -impl Default for TokenType { - fn default() -> TokenType { - Self::UrnColonIetfColonParamsColonOauthColonTokenTypeColonAccessToken - } -} diff --git a/crates/sail-catalog-iceberg/src/models/transform_term.rs b/crates/sail-catalog-iceberg/src/models/transform_term.rs deleted file mode 100644 index 4c7828ef1..000000000 --- a/crates/sail-catalog-iceberg/src/models/transform_term.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct TransformTerm { - #[serde(rename = "type")] - pub r#type: Type, - #[serde(rename = "transform")] - pub transform: String, - #[serde(rename = "term")] - pub term: String, -} - -impl TransformTerm { - pub fn new(r#type: Type, transform: String, term: String) -> TransformTerm { - TransformTerm { - r#type, - transform, - term, - } - } -} -/// -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub enum Type { - #[serde(rename = "transform")] - Transform, -} - -impl Default for Type { - fn default() -> Type { - Self::Transform - } -} diff --git a/crates/sail-catalog-iceberg/src/models/true_expression.rs b/crates/sail-catalog-iceberg/src/models/true_expression.rs deleted file mode 100644 index e596efe09..000000000 --- a/crates/sail-catalog-iceberg/src/models/true_expression.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct TrueExpression { - #[serde(rename = "type")] - pub r#type: String, -} - -impl TrueExpression { - pub fn new(r#type: String) -> TrueExpression { - TrueExpression { r#type } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/unary_expression.rs b/crates/sail-catalog-iceberg/src/models/unary_expression.rs deleted file mode 100644 index a1afd45f4..000000000 --- a/crates/sail-catalog-iceberg/src/models/unary_expression.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct UnaryExpression { - #[serde(rename = "type")] - pub r#type: String, - #[serde(rename = "term")] - pub term: Box, - #[serde(rename = "value")] - pub value: serde_json::Value, -} - -impl UnaryExpression { - pub fn new(r#type: String, term: models::Term, value: serde_json::Value) -> UnaryExpression { - UnaryExpression { - r#type, - term: Box::new(term), - value, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/update_namespace_properties_request.rs b/crates/sail-catalog-iceberg/src/models/update_namespace_properties_request.rs deleted file mode 100644 index ec91a0f0f..000000000 --- a/crates/sail-catalog-iceberg/src/models/update_namespace_properties_request.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct UpdateNamespacePropertiesRequest { - #[serde(rename = "removals", skip_serializing_if = "Option::is_none")] - pub removals: Option>, - #[serde(rename = "updates", skip_serializing_if = "Option::is_none")] - pub updates: Option>, -} - -impl UpdateNamespacePropertiesRequest { - pub fn new() -> UpdateNamespacePropertiesRequest { - UpdateNamespacePropertiesRequest { - removals: None, - updates: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/update_namespace_properties_response.rs b/crates/sail-catalog-iceberg/src/models/update_namespace_properties_response.rs deleted file mode 100644 index 34f8e0c51..000000000 --- a/crates/sail-catalog-iceberg/src/models/update_namespace_properties_response.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct UpdateNamespacePropertiesResponse { - /// List of property keys that were added or updated - #[serde(rename = "updated")] - pub updated: Vec, - /// List of properties that were removed - #[serde(rename = "removed")] - pub removed: Vec, - /// List of properties requested for removal that were not found in the namespace's properties. Represents a partial success response. Server's do not need to implement this. - #[serde(rename = "missing", skip_serializing_if = "Option::is_none")] - pub missing: Option>, -} - -impl UpdateNamespacePropertiesResponse { - pub fn new(updated: Vec, removed: Vec) -> UpdateNamespacePropertiesResponse { - UpdateNamespacePropertiesResponse { - updated, - removed, - missing: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/upgrade_format_version_update.rs b/crates/sail-catalog-iceberg/src/models/upgrade_format_version_update.rs deleted file mode 100644 index e6b131f81..000000000 --- a/crates/sail-catalog-iceberg/src/models/upgrade_format_version_update.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct UpgradeFormatVersionUpdate { - #[serde(rename = "action", skip_serializing_if = "Option::is_none")] - pub action: Option, - #[serde(rename = "format-version")] - pub format_version: i32, -} - -impl UpgradeFormatVersionUpdate { - pub fn new(format_version: i32) -> UpgradeFormatVersionUpdate { - UpgradeFormatVersionUpdate { - action: None, - format_version, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/value_map.rs b/crates/sail-catalog-iceberg/src/models/value_map.rs deleted file mode 100644 index 9be4a1983..000000000 --- a/crates/sail-catalog-iceberg/src/models/value_map.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ValueMap { - /// List of integer column ids for each corresponding value - #[serde(rename = "keys", skip_serializing_if = "Option::is_none")] - pub keys: Option>, - /// List of primitive type values, matched to 'keys' by index - #[serde(rename = "values", skip_serializing_if = "Option::is_none")] - pub values: Option>, -} - -impl ValueMap { - pub fn new() -> ValueMap { - ValueMap { - keys: None, - values: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/view_history_entry.rs b/crates/sail-catalog-iceberg/src/models/view_history_entry.rs deleted file mode 100644 index 697201998..000000000 --- a/crates/sail-catalog-iceberg/src/models/view_history_entry.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ViewHistoryEntry { - #[serde(rename = "version-id")] - pub version_id: i32, - #[serde(rename = "timestamp-ms")] - pub timestamp_ms: i64, -} - -impl ViewHistoryEntry { - pub fn new(version_id: i32, timestamp_ms: i64) -> ViewHistoryEntry { - ViewHistoryEntry { - version_id, - timestamp_ms, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/view_metadata.rs b/crates/sail-catalog-iceberg/src/models/view_metadata.rs deleted file mode 100644 index 13b43ef35..000000000 --- a/crates/sail-catalog-iceberg/src/models/view_metadata.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ViewMetadata { - #[serde(rename = "view-uuid")] - pub view_uuid: String, - #[serde(rename = "format-version")] - pub format_version: i32, - #[serde(rename = "location")] - pub location: String, - #[serde(rename = "current-version-id")] - pub current_version_id: i32, - #[serde(rename = "versions")] - pub versions: Vec, - #[serde(rename = "version-log")] - pub version_log: Vec, - #[serde(rename = "schemas")] - pub schemas: Vec, - #[serde(rename = "properties", skip_serializing_if = "Option::is_none")] - pub properties: Option>, -} - -impl ViewMetadata { - pub fn new( - view_uuid: String, - format_version: i32, - location: String, - current_version_id: i32, - versions: Vec, - version_log: Vec, - schemas: Vec, - ) -> ViewMetadata { - ViewMetadata { - view_uuid, - format_version, - location, - current_version_id, - versions, - version_log, - schemas, - properties: None, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/models/view_update.rs b/crates/sail-catalog-iceberg/src/models/view_update.rs deleted file mode 100644 index b5eb570c6..000000000 --- a/crates/sail-catalog-iceberg/src/models/view_update.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "action")] -pub enum ViewUpdate { - #[serde(rename = "add-schema")] - AddSchemaUpdate {}, - #[serde(rename = "add-view-version")] - AddViewVersionUpdate {}, - #[serde(rename = "assign-uuid")] - AssignUuidUpdate {}, - #[serde(rename = "remove-properties")] - RemovePropertiesUpdate {}, - #[serde(rename = "set-current-view-version")] - SetCurrentViewVersionUpdate {}, - #[serde(rename = "set-location")] - SetLocationUpdate {}, - #[serde(rename = "set-properties")] - SetPropertiesUpdate {}, - #[serde(rename = "upgrade-format-version")] - UpgradeFormatVersionUpdate {}, -} - -impl Default for ViewUpdate { - fn default() -> Self { - Self::AddSchemaUpdate {} - } -} diff --git a/crates/sail-catalog-iceberg/src/models/view_version.rs b/crates/sail-catalog-iceberg/src/models/view_version.rs deleted file mode 100644 index 7894af6a1..000000000 --- a/crates/sail-catalog-iceberg/src/models/view_version.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* - * Apache Iceberg REST Catalog API - * - * Defines the specification for the first version of the REST Catalog API. Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use serde::{Deserialize, Serialize}; - -use crate::models; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct ViewVersion { - #[serde(rename = "version-id")] - pub version_id: i32, - #[serde(rename = "timestamp-ms")] - pub timestamp_ms: i64, - /// Schema ID to set as current, or -1 to set last added schema - #[serde(rename = "schema-id")] - pub schema_id: i32, - #[serde(rename = "summary")] - pub summary: std::collections::HashMap, - #[serde(rename = "representations")] - pub representations: Vec, - #[serde(rename = "default-catalog", skip_serializing_if = "Option::is_none")] - pub default_catalog: Option, - /// Reference to one or more levels of a namespace - #[serde(rename = "default-namespace")] - pub default_namespace: Vec, -} - -impl ViewVersion { - pub fn new( - version_id: i32, - timestamp_ms: i64, - schema_id: i32, - summary: std::collections::HashMap, - representations: Vec, - default_namespace: Vec, - ) -> ViewVersion { - ViewVersion { - version_id, - timestamp_ms, - schema_id, - summary, - representations, - default_catalog: None, - default_namespace, - } - } -} diff --git a/crates/sail-catalog-iceberg/src/provider.rs b/crates/sail-catalog-iceberg/src/provider.rs index 92ff41380..82d951245 100644 --- a/crates/sail-catalog-iceberg/src/provider.rs +++ b/crates/sail-catalog-iceberg/src/provider.rs @@ -10,11 +10,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::Arc; use percent_encoding::percent_decode_str; +use sail_catalog::credentials::CatalogCredentials; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::lakehouse::{ BeginTableAccessRequest, LakehouseCapability, LakehouseCommitOutcome, LakehouseCommitRequest, @@ -22,11 +24,11 @@ use sail_catalog::lakehouse::{ }; use sail_catalog::provider::{ AlterTableOptions, CatalogPartitionField, CatalogProvider, CreateDatabaseOptions, - CreateTableColumnOptions, CreateTableMode, CreateTableOptions, CreateViewColumnOptions, - CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, - PartitionTransform, + CreateTableColumnOptions, CreateTableOptions, CreateViewColumnOptions, CreateViewOptions, + DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, PartitionTransform, }; use sail_catalog::utils::{get_property, quote_name_if_needed, quote_namespace_if_needed}; +use sail_common::utils::http::SAIL_USER_AGENT; use sail_common_datafusion::catalog::managed::METADATA_LOCATION_KEY; use sail_common_datafusion::catalog::{ CapabilityFingerprint, CatalogTableBucketBy, CatalogTableConstraint, CatalogTableSort, @@ -35,12 +37,11 @@ use sail_common_datafusion::catalog::{ }; use sail_iceberg::utils::partition_transform::catalog_partition_field_from_iceberg; use sail_iceberg::{ - arrow_type_to_iceberg, iceberg_type_to_arrow, FormatVersion, Literal, NestedField, StructType, + FormatVersion, Literal, NestedField, StructType, arrow_type_to_iceberg, iceberg_type_to_arrow, }; use tokio::sync::OnceCell; -use crate::apis::configuration::Configuration; -use crate::apis::{self, Api, ApiClient}; +use crate::r#gen::{ApiClient, ApiError}; pub const REST_CATALOG_PROP_URI: &str = "uri"; @@ -59,165 +60,35 @@ const REST_REMOTE_SIGNING_ENABLED_KEY: &str = "s3.remote-signing-enabled"; // - https://iceberg.apache.org/docs/nightly/configuration/#catalog-properties // - https://iceberg.apache.org/docs/nightly/spark-configuration/ #[derive(Clone, Debug)] -pub struct RestCatalogConfig { - uri: String, - warehouse: Option, - props: HashMap, +pub struct CatalogConfig<'a> { + properties: Cow<'a, HashMap>, } -/// Provider for Apache Iceberg REST Catalog. -pub struct IcebergRestCatalogProvider { - name: String, - catalog_config: RestCatalogConfig, - merged_catalog_config: OnceCell, - client: OnceCell, -} +impl CatalogConfig<'_> { + fn uri(&self) -> Option { + self.properties + .get(REST_CATALOG_PROP_URI) + .cloned() + .map(|value| value.trim_end_matches('/').to_string()) + .filter(|value| !value.is_empty()) + } -impl IcebergRestCatalogProvider { - fn normalize_catalog_props(props: HashMap) -> HashMap { - props - .into_iter() - .map(|(key, value)| { - let key = if Self::is_namespace_separator_key(&key) { - REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string() - } else { - key - }; - (key, value) - }) - .collect() + fn warehouse(&self) -> Option { + self.properties + .get(REST_CATALOG_PROP_WAREHOUSE) + .map(|value| value.trim_end_matches('/').to_string()) } - fn catalog_prefix(catalog_config: &RestCatalogConfig) -> Option<&str> { - catalog_config - .props + fn prefix(&self) -> Option<&str> { + self.properties .get(REST_CATALOG_PROP_PREFIX) .map(|value| value.trim()) .filter(|value| !value.is_empty()) } - fn is_namespace_separator_key(key: &str) -> bool { - matches!( - key.trim().to_ascii_lowercase().as_str(), - "namespace-separator" | "namespaceseparator" | "namespace_separator" - ) - } - - pub fn new(name: String, props: HashMap) -> Self { - let catalog_config = RestCatalogConfig { - uri: props - .get(REST_CATALOG_PROP_URI) - .cloned() - .unwrap_or(Configuration::new().base_path), - warehouse: props.get(REST_CATALOG_PROP_WAREHOUSE).cloned(), - props: Self::normalize_catalog_props( - props - .into_iter() - .filter(|(k, _)| k != REST_CATALOG_PROP_URI && k != REST_CATALOG_PROP_WAREHOUSE) - .collect(), - ), - }; - - Self { - name, - catalog_config, - merged_catalog_config: OnceCell::new(), - client: OnceCell::new(), - } - } - - fn init_client_config( - &self, - catalog_config: &RestCatalogConfig, - ) -> CatalogResult { - let mut client_config = Configuration::new(); - client_config.user_agent = Some("Sail".to_string()); - client_config.base_path = catalog_config.uri.to_string(); - for (key, value) in &catalog_config.props { - // TODO: `basic_auth` and `api_key` are not used anything in the API yet. - // We may need to support them in the future. - match key.as_str() { - "oauth-access-token" => { - client_config.oauth_access_token = Some(value.to_string()); - } - "bearer-access-token" => { - client_config.bearer_access_token = Some(value.to_string()); - } - _ => {} - } - } - Ok(client_config) - } - - fn init_client(&self, catalog_config: &RestCatalogConfig) -> CatalogResult { - Ok(ApiClient::new(Arc::new( - self.init_client_config(catalog_config)?, - ))) - } - - async fn load_catalog_config( - &self, - client: &ApiClient, - warehouse: Option<&str>, - ) -> CatalogResult { - let config = client - .configuration_api_api() - .get_config(warehouse) - .await - .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; - Ok(config) - } - - // Merge the `RestCatalogConfig` with the a [`CatalogConfig`] (fetched from the REST server). - // Then initialize the `ApiClient` with the merged config. - // This only happens once, then the result is cached. - async fn load_client_and_merged_config( - &self, - ) -> CatalogResult<(&ApiClient, &RestCatalogConfig)> { - let merged_catalog_config = self - .merged_catalog_config - .get_or_try_init(|| async { - let temp_client = self.init_client(&self.catalog_config)?; - let config = self - .load_catalog_config(&temp_client, self.catalog_config.warehouse.as_deref()) - .await?; - - let mut overrides = Self::normalize_catalog_props(config.overrides); - let mut client_props = self.catalog_config.props.clone(); - if let Some(warehouse) = &self.catalog_config.warehouse { - client_props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), warehouse.clone()); - } - - let uri = if let Some(uri) = overrides.remove(REST_CATALOG_PROP_URI) { - uri.trim_end_matches('/').to_string() - } else { - self.catalog_config.uri.trim_end_matches('/').to_string() - }; - - let mut props = Self::normalize_catalog_props(config.defaults); - props.extend(client_props); - props.extend(overrides); - let warehouse = props.get(REST_CATALOG_PROP_WAREHOUSE).cloned(); - - Ok::<_, CatalogError>(RestCatalogConfig { - uri, - warehouse, - props, - }) - }) - .await?; - - let client = self - .client - .get_or_try_init(|| async { self.init_client(merged_catalog_config) }) - .await?; - - Ok((client, merged_catalog_config)) - } - - fn namespace_separator(catalog_config: &RestCatalogConfig) -> CatalogResult { - let separator = catalog_config - .props + fn namespace_separator(&self) -> CatalogResult { + let separator = self + .properties .get(REST_CATALOG_PROP_NAMESPACE_SEPARATOR) .map(|value| value.trim()); @@ -235,11 +106,8 @@ impl IcebergRestCatalogProvider { } /// Converts a `Namespace` into a string representation for the REST API URL. - fn namespace_string( - catalog_config: &RestCatalogConfig, - database: &Namespace, - ) -> CatalogResult { - let separator = Self::namespace_separator(catalog_config)?; + fn namespace_string(&self, database: &Namespace) -> CatalogResult { + let separator = self.namespace_separator()?; let mut result = database.head.to_string(); for s in &database.tail { result.push_str(&separator); @@ -247,160 +115,194 @@ impl IcebergRestCatalogProvider { } Ok(result) } +} - async fn load_table_result_with( - &self, - client: &ApiClient, - catalog_config: &RestCatalogConfig, - database: &Namespace, - table: &str, - access_delegation: Option<&str>, - ) -> CatalogResult { - client - .catalog_api_api() - .load_table( - &Self::namespace_string(catalog_config, database)?, - table, - access_delegation, - None, - None, - Self::catalog_prefix(catalog_config), - ) - .await - .map_err(|e| match e { - apis::Error::ResponseError(apis::ResponseContent { status, .. }) - if status == 404 => - { - CatalogError::NotFound( - CatalogObject::Table, - format!( - "{}.{}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ), - ) - } - _ => CatalogError::External(format!( - "Failed to load table {}.{}: {e}", - quote_namespace_if_needed(database), - quote_name_if_needed(table), - )), - }) - } +#[derive(Debug, Clone)] +pub struct IcebergRestCatalogOptions { + pub credentials: Arc, + pub properties: HashMap, +} - async fn load_table_result( - &self, - database: &Namespace, - table: &str, - ) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - self.load_table_result_with(client, catalog_config, database, table, None) - .await - } +/// Provider for Apache Iceberg REST Catalog. +pub struct IcebergRestCatalogProvider { + name: String, + options: IcebergRestCatalogOptions, + http_client: reqwest::Client, + resolved_catalog_config: OnceCell>, +} - async fn commit_table_request( - &self, - database: &Namespace, - table: &str, - format: &str, - requirements: Vec, - updates: Vec, - ) -> CatalogResult { - if !format.eq_ignore_ascii_case("iceberg") { - return Err(CatalogError::NotSupported(format!( - "Iceberg REST catalog cannot commit '{format}' tables", - ))); +impl IcebergRestCatalogProvider { + pub fn new(name: String, options: IcebergRestCatalogOptions) -> Self { + Self { + name, + options, + http_client: reqwest::Client::new(), + resolved_catalog_config: OnceCell::new(), } + } - let (_client, catalog_config) = self.load_client_and_merged_config().await?; - let client_config = self.init_client_config(catalog_config)?; - let namespace = Self::namespace_string(catalog_config, database)?; - let prefix = catalog_config - .props - .get(REST_CATALOG_PROP_PREFIX) - .map(|prefix| format!("/{}", crate::apis::urlencode(prefix))) - .unwrap_or_default(); - let uri = format!( - "{}/v1{prefix}/namespaces/{namespace}/tables/{table}", - client_config.base_path, - namespace = crate::apis::urlencode(namespace), - table = crate::apis::urlencode(table) + async fn configuration( + catalog_config: &CatalogConfig<'_>, + credentials: &Arc, + http_client: reqwest::Client, + ) -> CatalogResult { + let base_path = catalog_config.uri().ok_or_else(|| { + CatalogError::InvalidArgument(format!( + "Iceberg REST catalog property '{REST_CATALOG_PROP_URI}' is required" + )) + })?; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static(SAIL_USER_AGENT), ); - let body = serde_json::json!({ - "identifier": { - "namespace": Vec::::from(database.clone()), - "name": table, - }, - "requirements": requirements, - "updates": updates, - }); - - let mut request = client_config.client.post(uri); - if let Some(user_agent) = client_config.user_agent { - request = request.header(reqwest::header::USER_AGENT, user_agent); + if let Some(credential) = credentials.retrieve().await? { + let header = reqwest::header::HeaderValue::from_str(&format!("Bearer {credential}")) + .map_err(|e| { + CatalogError::External(format!("Failed to create header value from token: {e}")) + })?; + headers.insert(reqwest::header::AUTHORIZATION, header); } - if let Some(token) = client_config.oauth_access_token { - request = request.bearer_auth(token); + Ok(ApiClient::new(base_path, http_client, headers)) + } + + async fn bootstrap_client(&self) -> CatalogResult { + let catalog_config = CatalogConfig { + properties: Cow::Borrowed(&self.options.properties), + }; + Self::configuration( + &catalog_config, + &self.options.credentials, + self.http_client.clone(), + ) + .await + } + + async fn client(&self) -> CatalogResult { + let catalog_config = self.resolved_catalog_config().await?; + Self::configuration( + catalog_config, + &self.options.credentials, + self.http_client.clone(), + ) + .await + } + + /// Retry the bootstrap configuration request once on `401 Unauthorized`. + /// This uses [`Self::bootstrap_client`] rather than [`Self::client`] + /// because the resolved client itself depends on the bootstrap response. + /// Each attempt reloads the credential while reusing the shared HTTP + /// client and its connection pool. + async fn with_bootstrap_auth_retry( + &self, + call: F, + ) -> CatalogResult>> + where + F: Fn(ApiClient) -> Fut, + Fut: std::future::Future>>, + { + let client = self.bootstrap_client().await?; + let result = call(client).await; + if matches!(&result, Err(e) if e.status() == Some(reqwest::StatusCode::UNAUTHORIZED)) { + let client = self.bootstrap_client().await?; + return Ok(call(client).await); } - if let Some(token) = client_config.bearer_access_token { - request = request.bearer_auth(token); + Ok(result) + } + + /// Run a single outbound REST request, retrying it once if the server + /// answers `401 Unauthorized`. Each attempt builds an [`ApiClient`] from a + /// freshly resolved credential, so a projected service account token that + /// rotated mid-operation is picked up on the retry. The credential is + /// re-read per request, so every request in a `drop_database` cascade sees + /// the current token. The shared `reqwest::Client` and its connection pool + /// are reused across attempts. + async fn with_auth_retry(&self, call: F) -> CatalogResult>> + where + F: Fn(ApiClient) -> Fut, + Fut: std::future::Future>>, + { + let client = self.client().await?; + let result = call(client).await; + if matches!(&result, Err(e) if e.status() == Some(reqwest::StatusCode::UNAUTHORIZED)) { + let client = self.client().await?; + return Ok(call(client).await); } + Ok(result) + } - let response = request - .json(&body) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to commit table: {e}")))?; - let status = response.status(); - let content = response - .text() - .await - .map_err(|e| CatalogError::External(format!("Failed to read commit response: {e}")))?; + // Merge the local catalog config with the [`crate::r#gen::CatalogConfig`] fetched from the REST server. + // This only happens once, then the result is cached. + async fn resolved_catalog_config(&self) -> CatalogResult<&CatalogConfig<'static>> { + self.resolved_catalog_config + .get_or_try_init(|| async { + let catalog_config = CatalogConfig { + properties: Cow::Borrowed(&self.options.properties), + }; + let warehouse = catalog_config.warehouse(); + let config = self + .with_bootstrap_auth_retry(|client| { + let warehouse = warehouse.clone(); + async move { client.get_config(warehouse).await } + }) + .await? + .map(|response| response.inner) + .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; + + let mut properties = config + .defaults + .into_iter() + .filter(|(key, _)| key != REST_CATALOG_PROP_URI) + .collect::>(); + properties.extend(self.options.properties.clone()); + properties.extend(config.overrides); - if status.is_success() { - serde_json::from_str(&content).map_err(|e| { - CatalogError::External(format!("Failed to parse commit response: {e}")) + Ok::<_, CatalogError>(CatalogConfig { + properties: Cow::Owned(properties), + }) }) - } else if status == reqwest::StatusCode::NOT_FOUND { - Err(CatalogError::NotFound( + .await + } + + async fn load_table_result( + &self, + database: &Namespace, + table: &str, + access_delegation: Option<&str>, + ) -> CatalogResult { + let catalog_config = self.resolved_catalog_config().await?; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + let table_name = table.to_string(); + let access_delegation = access_delegation.map(ToOwned::to_owned); + self.with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let table_name = table_name.clone(); + let access_delegation = access_delegation.clone(); + async move { + client + .load_table(prefix, namespace, table_name, access_delegation, None, None) + .await + } + }) + .await? + .map(|response| response.inner) + .map_err(|e| match e { + e if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => CatalogError::NotFound( CatalogObject::Table, format!( "{}.{}", quote_namespace_if_needed(database), quote_name_if_needed(table) ), - )) - } else if status == reqwest::StatusCode::CONFLICT { - Err(CatalogError::Conflict(format!( - "Iceberg REST catalog commit conflict for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else if status == reqwest::StatusCode::UNAUTHORIZED { - Err(CatalogError::Unauthorized(format!( - "Iceberg REST catalog commit unauthorized for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else if status == reqwest::StatusCode::FORBIDDEN { - Err(CatalogError::Forbidden(format!( - "Iceberg REST catalog commit forbidden for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else if status == reqwest::StatusCode::TOO_MANY_REQUESTS { - Err(CatalogError::RateLimited(format!( - "Iceberg REST catalog commit rate limited for {}.{}: {content}", - quote_namespace_if_needed(database), - quote_name_if_needed(table) - ))) - } else { - Err(CatalogError::External(format!( - "Failed to commit Iceberg table {}.{}: status {status}: {content}", + ), + _ => CatalogError::External(format!( + "Failed to load table {}.{}: {e}", quote_namespace_if_needed(database), quote_name_if_needed(table) - ))) - } + )), + }) } fn normalize_scan_planning_mode(value: &str) -> CatalogResult { @@ -415,29 +317,40 @@ impl IcebergRestCatalogProvider { fn effective_scan_planning_mode( table_config: Option<&HashMap>, - catalog_config: &RestCatalogConfig, + catalog_config: &CatalogConfig<'_>, ) -> CatalogResult> { table_config .and_then(|config| config.get(REST_TABLE_SCAN_PLANNING_MODE_KEY)) - .or_else(|| catalog_config.props.get(REST_TABLE_SCAN_PLANNING_MODE_KEY)) + .or_else(|| { + catalog_config + .properties + .get(REST_TABLE_SCAN_PLANNING_MODE_KEY) + }) .map(|value| Self::normalize_scan_planning_mode(value)) .transpose() } fn remote_signing_enabled( table_config: Option<&HashMap>, - catalog_config: &RestCatalogConfig, + catalog_config: &CatalogConfig<'_>, ) -> bool { table_config .and_then(|config| config.get(REST_REMOTE_SIGNING_ENABLED_KEY)) - .or_else(|| catalog_config.props.get(REST_REMOTE_SIGNING_ENABLED_KEY)) + .or_else(|| { + catalog_config + .properties + .get(REST_REMOTE_SIGNING_ENABLED_KEY) + }) .map(|value| value.eq_ignore_ascii_case("true")) .unwrap_or(false) } - fn hash_string_map(map: &HashMap, hasher: &mut impl Hasher) { - let mut entries = map.iter().collect::>(); - entries.sort_by(|(left_key, _), (right_key, _)| left_key.cmp(right_key)); + fn hash_string_map<'a>( + entries: impl IntoIterator, + hasher: &mut impl Hasher, + ) { + let mut entries = entries.into_iter().collect::>(); + entries.sort_by_key(|(key, _)| *key); for (key, value) in entries { key.hash(hasher); value.hash(hasher); @@ -445,23 +358,24 @@ impl IcebergRestCatalogProvider { } fn rest_table_session_fingerprint( - &self, + catalog: &str, database: &Namespace, table: &str, scan_planning_mode: Option<&str>, remote_signing_enabled: bool, - result: &crate::models::LoadTableResult, + result: &crate::r#gen::LoadTableResult, ) -> String { let mut hasher = DefaultHasher::new(); - self.name.hash(&mut hasher); - Vec::::from(database.clone()).hash(&mut hasher); + catalog.hash(&mut hasher); + let namespace: Vec = database.clone().into(); + namespace.hash(&mut hasher); table.hash(&mut hasher); result.metadata_location.hash(&mut hasher); result.metadata.table_uuid.hash(&mut hasher); scan_planning_mode.hash(&mut hasher); remote_signing_enabled.hash(&mut hasher); if let Some(config) = result.config.as_ref() { - Self::hash_string_map(config, &mut hasher); + Self::hash_string_map(config.iter(), &mut hasher); } if let Some(credentials) = result.storage_credentials.as_ref() { let mut summaries = credentials @@ -469,28 +383,29 @@ impl IcebergRestCatalogProvider { .map(|credential| { let mut credential_hasher = DefaultHasher::new(); credential.prefix.hash(&mut credential_hasher); - Self::hash_string_map(&credential.config, &mut credential_hasher); + Self::hash_string_map(credential.config.iter(), &mut credential_hasher); (credential.prefix.as_str(), credential_hasher.finish()) }) .collect::>(); - summaries.sort_by(|(left_prefix, _), (right_prefix, _)| left_prefix.cmp(right_prefix)); + summaries.sort_by_key(|(prefix, _)| *prefix); summaries.hash(&mut hasher); } format!("iceberg-rest:{:016x}", hasher.finish()) } fn rest_table_session_ref( - &self, + catalog: &str, database: &Namespace, table: &str, - catalog_config: &RestCatalogConfig, - result: &crate::models::LoadTableResult, + catalog_config: &CatalogConfig<'_>, + result: &crate::r#gen::LoadTableResult, ) -> CatalogResult { let config = result.config.as_ref(); let credentials = result.storage_credentials.as_deref(); let scan_planning_mode = Self::effective_scan_planning_mode(config, catalog_config)?; let remote_signing_enabled = Self::remote_signing_enabled(config, catalog_config); - let fingerprint = self.rest_table_session_fingerprint( + let fingerprint = Self::rest_table_session_fingerprint( + catalog, database, table, scan_planning_mode.as_deref(), @@ -508,45 +423,63 @@ impl IcebergRestCatalogProvider { } fn validate_create_table_access_session_requirements( - &self, + catalog: &str, database: &Namespace, table: &str, - catalog_config: &RestCatalogConfig, - result: &crate::models::LoadTableResult, + catalog_config: &CatalogConfig<'_>, + result: &crate::r#gen::LoadTableResult, ) -> CatalogResult<()> { - let rest_session = self.rest_table_session_ref(database, table, catalog_config, result)?; - let mut requirements = Vec::new(); - if rest_session.scan_planning_mode.as_deref() == Some("server") { - requirements.push("server-side scan planning"); + let scan_planning_mode = + Self::effective_scan_planning_mode(result.config.as_ref(), catalog_config)?; + if scan_planning_mode.as_deref() == Some("server") { + return Err(CatalogError::UnsupportedCapability( + "Iceberg REST access session requirements returned by create_table are not supported for create+write yet: server-side scan planning".to_string(), + )); } - if rest_session.remote_signing_enabled { - requirements.push("remote signing"); + + let mut configured_storage_fallbacks = Vec::new(); + if Self::remote_signing_enabled(result.config.as_ref(), catalog_config) { + configured_storage_fallbacks.push("remote signing"); } - if rest_session.storage_credential_count > 0 { - requirements.push("vended credentials"); + if result + .storage_credentials + .as_ref() + .is_some_and(|credentials| !credentials.is_empty()) + { + configured_storage_fallbacks.push("vended credentials"); } - - if requirements.is_empty() { - Ok(()) - } else { - Err(CatalogError::UnsupportedCapability(format!( - "Iceberg REST access session requirements returned by create_table are not supported for create+write yet: {}", - requirements.join(", ") - ))) + if !configured_storage_fallbacks.is_empty() { + log::warn!( + "Iceberg REST catalog {} create_table for {}.{} returned {}; using configured object-store credentials for create+write", + catalog, + quote_namespace_if_needed(database), + quote_name_if_needed(table), + configured_storage_fallbacks.join(", "), + ); } + + Ok(()) } /// Converts an Iceberg REST API table load result into a catalog `TableStatus`. fn load_table_result_to_status( - &self, - table_name: &str, + catalog: &str, database: &Namespace, - result: crate::models::LoadTableResult, + table: &str, + result: crate::r#gen::LoadTableResult, ) -> CatalogResult { + log::trace!( + "Iceberg REST table load result: catalog={}, database={:?}, table={}, metadata.location={:?}, metadata-location={:?}", + catalog, + database, + table, + result.metadata.location, + result.metadata_location, + ); // Table-specific config and storage credentials are access-session state, not // display table properties. // TODO: Preserve unused fields in `TableMetadata` when Sail exposes them. - let crate::models::TableMetadata { + let crate::r#gen::TableMetadata { format_version, table_uuid, location, @@ -585,7 +518,7 @@ impl IcebergRestCatalogProvider { .map(|spec| { spec.fields .iter() - .filter(|f| f.transform.trim().to_lowercase().starts_with("bucket")) + .filter(|f| f.transform.0.trim().to_lowercase().starts_with("bucket")) .map(|f| f.source_id) .collect() }) @@ -608,7 +541,7 @@ impl IcebergRestCatalogProvider { })? .name .clone(); - let transform = field.transform.parse().map_err(CatalogError::External)?; + let transform = field.transform.0.parse().map_err(CatalogError::External)?; catalog_partition_field_from_iceberg(source_column, transform) .map_err(CatalogError::External) }) @@ -619,7 +552,9 @@ impl IcebergRestCatalogProvider { let columns = if let Some(schema) = current_schema { let mut cols = Vec::new(); for field in &schema.fields { - let data_type = iceberg_type_to_arrow(&field.field_type).map_err(|e| { + let iceberg_type = + sail_iceberg::spec::types::Type::try_from(field.r#type.as_ref().clone())?; + let data_type = iceberg_type_to_arrow(&iceberg_type).map_err(|e| { CatalogError::External(format!( "Failed to convert Iceberg type to Arrow type for field '{}': {e}", field.name @@ -662,9 +597,9 @@ impl IcebergRestCatalogProvider { .iter() .find(|f| f.id == field_id) .map(|field| { - let ascending = match sort_field.direction { - crate::models::SortDirection::Asc => true, - crate::models::SortDirection::Desc => false, + let ascending = match *sort_field.direction { + crate::r#gen::SortDirection::Asc => true, + crate::r#gen::SortDirection::Desc => false, }; CatalogTableSort { column: field.name.clone(), @@ -762,9 +697,9 @@ impl IcebergRestCatalogProvider { let properties: Vec<_> = properties.into_iter().collect(); Ok(TableStatus { - catalog: Some(self.name.clone()), + catalog: Some(catalog.to_string()), database: database.clone().into(), - name: table_name.to_string(), + name: table.to_string(), kind: TableKind::Table { columns, comment, @@ -781,15 +716,15 @@ impl IcebergRestCatalogProvider { } fn load_view_result_to_status( - &self, - view_name: &str, + catalog: &str, database: &Namespace, - result: crate::models::LoadViewResult, + view: &str, + result: crate::r#gen::LoadViewResult, ) -> CatalogResult { // TODO: Do we want to do anything with: // - `result.config`` // - Unused fields in `ViewMetadata`? - let crate::models::ViewMetadata { + let crate::r#gen::ViewMetadata { view_uuid, format_version, location, @@ -813,7 +748,9 @@ impl IcebergRestCatalogProvider { let columns = if let Some(schema) = current_schema { let mut cols = Vec::new(); for field in &schema.fields { - let data_type = iceberg_type_to_arrow(&field.field_type).map_err(|e| { + let iceberg_type = + sail_iceberg::spec::types::Type::try_from(field.r#type.as_ref().clone())?; + let data_type = iceberg_type_to_arrow(&iceberg_type).map_err(|e| { CatalogError::External(format!( "Failed to convert Iceberg type to Arrow type for field '{}': {e}", field.name @@ -841,10 +778,16 @@ impl IcebergRestCatalogProvider { .and_then(|v| { v.representations .iter() - .find(|r| r.dialect.trim().to_lowercase() == "spark") + .find(|r| match r { + crate::r#gen::ViewRepresentation::SqlViewRepresentation(r) => { + r.dialect.trim().to_lowercase() == "spark" + } + }) .or_else(|| v.representations.last()) }) - .map(|r| r.sql.clone()) + .map(|r| match r { + crate::r#gen::ViewRepresentation::SqlViewRepresentation(r) => r.sql.clone(), + }) .unwrap_or_default(); let mut properties: HashMap = properties.unwrap_or_default(); @@ -866,9 +809,9 @@ impl IcebergRestCatalogProvider { let properties: Vec<_> = properties.into_iter().collect(); Ok(TableStatus { - catalog: Some(self.name.clone()), + catalog: Some(catalog.to_string()), database: database.clone().into(), - name: view_name.to_string(), + name: view.to_string(), kind: TableKind::View { definition, columns, @@ -897,7 +840,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { database: &Namespace, options: CreateDatabaseOptions, ) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let CreateDatabaseOptions { if_not_exists, @@ -914,15 +857,20 @@ impl CatalogProvider for IcebergRestCatalogProvider { props.insert("location".to_string(), l); } - let request = crate::models::CreateNamespaceRequest { - namespace: database.clone().into(), + let request = crate::r#gen::CreateNamespaceRequest { + namespace: Box::new(database.clone().into()), properties: if props.is_empty() { None } else { Some(props) }, }; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); - let result = client - .catalog_api_api() - .create_namespace(request, Self::catalog_prefix(catalog_config)) - .await; + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let request = request.clone(); + async move { client.create_namespace(prefix, request).await } + }) + .await? + .map(|response| response.inner); match result { Ok(result) => { @@ -939,15 +887,13 @@ impl CatalogProvider for IcebergRestCatalogProvider { Ok(DatabaseStatus { catalog: self.name.clone(), - database: result.namespace, + database: (*result.namespace).into(), comment, location, properties, }) } - Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 409 && if_not_exists => - { + Err(e) if e.status() == Some(reqwest::StatusCode::CONFLICT) && if_not_exists => { self.get_database(database).await } Err(e) => Err(CatalogError::External(format!( @@ -957,27 +903,23 @@ impl CatalogProvider for IcebergRestCatalogProvider { } async fn get_database(&self, database: &Namespace) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - let namespace = Self::namespace_string(catalog_config, database)?; + let catalog_config = self.resolved_catalog_config().await?; + let namespace = catalog_config.namespace_string(database)?; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); - let result = client - .catalog_api_api() - .load_namespace_metadata(&namespace, Self::catalog_prefix(catalog_config)) - .await + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + async move { client.load_namespace_metadata(prefix, namespace).await } + }) + .await? + .map(|response| response.inner) .map_err(|e| match e { - apis::Error::ResponseError(apis::ResponseContent { status, .. }) => { - if status == 404 { - CatalogError::NotFound( - CatalogObject::Namespace, - quote_namespace_if_needed(database), - ) - } else { - CatalogError::External(format!( - "Failed to load namespace {}: {e}", - quote_namespace_if_needed(database) - )) - } - } + e if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => CatalogError::NotFound( + CatalogObject::Namespace, + quote_namespace_if_needed(database), + ), _ => CatalogError::External(format!( "Failed to load namespace {}: {e}", quote_namespace_if_needed(database) @@ -996,7 +938,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { Ok(DatabaseStatus { catalog: self.name.clone(), - database: result.namespace, + database: (*result.namespace).into(), comment, location, properties, @@ -1007,20 +949,24 @@ impl CatalogProvider for IcebergRestCatalogProvider { &self, prefix: Option<&Namespace>, ) -> CatalogResult> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let parent = prefix - .map(|namespace| Self::namespace_string(catalog_config, namespace)) + .map(|namespace| catalog_config.namespace_string(namespace)) .transpose()?; + let request_prefix = catalog_config.prefix().map(ToOwned::to_owned); - let result = client - .catalog_api_api() - .list_namespaces( - None, - None, - parent.as_deref(), - Self::catalog_prefix(catalog_config), - ) - .await + let result = self + .with_auth_retry(|client| { + let request_prefix = request_prefix.clone(); + let parent = parent.clone(); + async move { + client + .list_namespaces(request_prefix, None, None, parent) + .await + } + }) + .await? + .map(|response| response.inner) .map_err(|e| CatalogError::External(format!("Failed to list namespaces: {e}")))?; Ok(result @@ -1029,7 +975,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { .into_iter() .map(|namespace| DatabaseStatus { catalog: self.get_name().to_string(), - database: namespace, + database: namespace.into(), comment: None, location: None, properties: Vec::new(), @@ -1042,54 +988,67 @@ impl CatalogProvider for IcebergRestCatalogProvider { database: &Namespace, options: DropDatabaseOptions, ) -> CatalogResult<()> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let DropDatabaseOptions { if_exists, cascade } = options; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let ns_string = catalog_config.namespace_string(database)?; + if cascade { // For CASCADE, first drop all tables and views in the namespace before dropping the namespace. - let prefix = Self::catalog_prefix(catalog_config); - let ns_string = Self::namespace_string(catalog_config, database)?; - let tables_result = client - .catalog_api_api() - .list_tables(&ns_string, None, None, prefix) - .await; + // Each request re-reads the credential and retries once on a 401, so a token that rotates + // partway through the cascade is recovered per request instead of leaving a partial drop. + let tables_result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let ns_string = ns_string.clone(); + async move { client.list_tables(prefix, ns_string, None, None).await } + }) + .await?; if let Ok(tables) = tables_result { - for identifier in tables.identifiers.unwrap_or_default() { - let _ = client - .catalog_api_api() - .drop_table(&ns_string, &identifier.name, Some(true), prefix) - .await; + for identifier in tables.inner.identifiers.unwrap_or_default() { + let _ = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let ns_string = ns_string.clone(); + let name = identifier.name.clone(); + async move { client.drop_table(prefix, ns_string, name, Some(true)).await } + }) + .await?; } } - let views_result = client - .catalog_api_api() - .list_views(&ns_string, None, None, prefix) - .await; + let views_result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let ns_string = ns_string.clone(); + async move { client.list_views(prefix, ns_string, None, None).await } + }) + .await?; if let Ok(views) = views_result { - for identifier in views.identifiers.unwrap_or_default() { - let _ = client - .catalog_api_api() - .drop_view(&ns_string, &identifier.name, prefix) - .await; + for identifier in views.inner.identifiers.unwrap_or_default() { + let _ = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let ns_string = ns_string.clone(); + let name = identifier.name.clone(); + async move { client.drop_view(prefix, ns_string, name).await } + }) + .await?; } } } - match client - .catalog_api_api() - .drop_namespace( - &Self::namespace_string(catalog_config, database)?, - Self::catalog_prefix(catalog_config), - ) - .await + match self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let ns_string = ns_string.clone(); + async move { client.drop_namespace(prefix, ns_string).await } + }) + .await? { Ok(_) => Ok(()), - Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 404 && if_exists => - { - Ok(()) - } + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) && if_exists => Ok(()), Err(e) => Err(CatalogError::External(format!( "Failed to drop namespace: {e}" ))), @@ -1123,12 +1082,12 @@ impl CatalogProvider for IcebergRestCatalogProvider { ))); } - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; - if mode.ignore_if_exists() { - if let Ok(existing) = self.get_table(database, table).await { - return Ok(existing); - } + if mode.ignore_if_exists() + && let Ok(existing) = self.get_table(database, table).await + { + return Ok(existing); } if mode.is_replace() { @@ -1164,12 +1123,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { .with_identifier_field_ids(identifier_field_ids.clone()) .build() .map_err(|e| CatalogError::External(format!("Failed to build schema: {e}")))?; - let schema = crate::models::Schema { - r#type: crate::models::schema::Type::Struct, - fields: schema.fields().to_vec(), - schema_id: Some(schema.schema_id()), - identifier_field_ids: Some(schema.identifier_field_ids().collect()), - }; + let schema = crate::r#gen::Schema::try_from(schema)?; let partition_spec = build_partition_spec(&partition_by, bucket_by.as_ref(), &name_to_id)?; let write_order = build_sort_order(&sort_by, &name_to_id)?; @@ -1182,7 +1136,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { props.insert(k, v); } - let request = crate::models::CreateTableRequest { + let request = crate::r#gen::CreateTableRequest { name: table.to_string(), location, schema: Box::new(schema), @@ -1192,45 +1146,49 @@ impl CatalogProvider for IcebergRestCatalogProvider { properties: if props.is_empty() { None } else { Some(props) }, }; - let result = client - .catalog_api_api() - .create_table( - &Self::namespace_string(catalog_config, database)?, - request, - None, - Self::catalog_prefix(catalog_config), - ) - .await + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let request = request.clone(); + async move { client.create_table(prefix, namespace, None, request).await } + }) + .await? + .map(|response| response.inner) .map_err(|e| CatalogError::External(format!("Failed to create table: {e}")))?; if is_write_precondition { - self.validate_create_table_access_session_requirements( + Self::validate_create_table_access_session_requirements( + &self.name, database, table, catalog_config, &result, )?; } - self.load_table_result_to_status(table, database, result) + Self::load_table_result_to_status(&self.name, database, table, result) } async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { - let result = self.load_table_result(database, table).await?; - self.load_table_result_to_status(table, database, result) + let result = self.load_table_result(database, table, None).await?; + Self::load_table_result_to_status(&self.name, database, table, result) } async fn list_tables(&self, database: &Namespace) -> CatalogResult> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - - let result = client - .catalog_api_api() - .list_tables( - &Self::namespace_string(catalog_config, database)?, - None, - None, - Self::catalog_prefix(catalog_config), - ) - .await + let catalog_config = self.resolved_catalog_config().await?; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + async move { client.list_tables(prefix, namespace, None, None).await } + }) + .await? + .map(|response| response.inner) .map_err(|e| CatalogError::External(format!("Failed to list tables: {e}")))?; Ok(result @@ -1239,7 +1197,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { .into_iter() .map(|identifier| TableStatus { catalog: Some(self.name.clone()), - database: identifier.namespace, + database: (*identifier.namespace).into(), name: identifier.name, kind: TableKind::Table { columns: Vec::new(), @@ -1263,24 +1221,26 @@ impl CatalogProvider for IcebergRestCatalogProvider { table: &str, options: DropTableOptions, ) -> CatalogResult<()> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let DropTableOptions { if_exists, purge } = options; - match client - .catalog_api_api() - .drop_table( - &Self::namespace_string(catalog_config, database)?, - table, - Some(purge), - Self::catalog_prefix(catalog_config), - ) - .await + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + let table_name = table.to_string(); + match self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let table_name = table_name.clone(); + async move { + client + .drop_table(prefix, namespace, table_name, Some(purge)) + .await + } + }) + .await? { Ok(_) => Ok(()), - Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 404 && if_exists => - { - Ok(()) - } + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) && if_exists => Ok(()), Err(e) => Err(CatalogError::External(format!("Failed to drop table: {e}"))), } } @@ -1309,9 +1269,98 @@ impl CatalogProvider for IcebergRestCatalogProvider { updates, payload, } = request; + if !format.eq_ignore_ascii_case("iceberg") { + return Err(CatalogError::NotSupported(format!( + "Iceberg REST catalog cannot commit '{format}' tables", + ))); + } + + let catalog_config = self.resolved_catalog_config().await?; + let namespace = catalog_config.namespace_string(database)?; + let requirements = requirements + .into_iter() + .map(serde_json::from_value) + .collect::, _>>() + .map_err(|e| { + CatalogError::External(format!( + "Failed to parse Iceberg REST commit requirements: {e}" + )) + })?; + let updates = updates + .into_iter() + .map(serde_json::from_value) + .collect::, _>>() + .map_err(|e| { + CatalogError::External(format!("Failed to parse Iceberg REST commit updates: {e}")) + })?; + let request = crate::r#gen::CommitTableRequest { + identifier: Some(Box::new(crate::r#gen::TableIdentifier { + namespace: Box::new(database.clone().into()), + name: table.to_string(), + })), + requirements, + updates, + }; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let table_name = table.to_string(); let response = self - .commit_table_request(database, table, &format, requirements, updates) - .await?; + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let table_name = table_name.clone(); + let request = request.clone(); + async move { + client + .update_table(prefix, namespace, table_name, request) + .await + } + }) + .await? + .map(|response| response.inner) + .map_err(|e| match e { + e if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => CatalogError::NotFound( + CatalogObject::Table, + format!( + "{}.{}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + ), + ), + e if e.status() == Some(reqwest::StatusCode::CONFLICT) => { + CatalogError::Conflict(format!( + "Iceberg REST catalog commit conflict for {}.{}: {e}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )) + } + e if e.status() == Some(reqwest::StatusCode::UNAUTHORIZED) => { + CatalogError::Unauthorized(format!( + "Iceberg REST catalog commit unauthorized for {}.{}: {e}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )) + } + e if e.status() == Some(reqwest::StatusCode::FORBIDDEN) => { + CatalogError::Forbidden(format!( + "Iceberg REST catalog commit forbidden for {}.{}: {e}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )) + } + e if e.status() == Some(reqwest::StatusCode::TOO_MANY_REQUESTS) => { + CatalogError::RateLimited(format!( + "Iceberg REST catalog commit rate limited for {}.{}: {e}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )) + } + e if e.status().is_some() => CatalogError::External(format!( + "Failed to commit Iceberg table {}.{}: {e}", + quote_namespace_if_needed(database), + quote_name_if_needed(table) + )), + e => CatalogError::External(format!("Failed to commit table: {e}")), + })?; let payload = match payload { Some(payload) => Some(payload), None => Some(serde_json::to_value(response).map_err(|e| { @@ -1331,11 +1380,9 @@ impl CatalogProvider for IcebergRestCatalogProvider { mut context, purpose: _, } = request; - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let result = self - .load_table_result_with( - client, - catalog_config, + .load_table_result( database, table, Some(REST_ACCESS_DELEGATION_VENDED_CREDENTIALS), @@ -1343,7 +1390,8 @@ impl CatalogProvider for IcebergRestCatalogProvider { .await?; // TODO: Convert preserved REST table-session credentials into operation-scoped // FileIO/object-store access instead of only fingerprinting the session. - let rest_session = self.rest_table_session_ref(database, table, catalog_config, &result)?; + let rest_session = + Self::rest_table_session_ref(&self.name, database, table, catalog_config, &result)?; if rest_session.scan_planning_mode.as_deref() == Some("server") { context.scan = ScanAuthority::IcebergRestServerSide; } else { @@ -1380,7 +1428,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { view: &str, options: CreateViewOptions, ) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let CreateViewOptions { columns, @@ -1391,16 +1439,16 @@ impl CatalogProvider for IcebergRestCatalogProvider { properties, } = options; - if if_not_exists || replace { - if let Ok(existing) = self.get_view(database, view).await { - if if_not_exists { - return Ok(existing); - } - if replace { - return Err(CatalogError::NotSupported( - "Replace view is not supported yet".to_string(), - )); - } + if (if_not_exists || replace) + && let Ok(existing) = self.get_view(database, view).await + { + if if_not_exists { + return Ok(existing); + } + if replace { + return Err(CatalogError::NotSupported( + "Replace view is not supported yet".to_string(), + )); } } @@ -1433,14 +1481,9 @@ impl CatalogProvider for IcebergRestCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to build schema: {e}")))?; let schema = sail_iceberg::SchemaEvolver::assign_schema_field_ids(&schema) .map_err(|e| CatalogError::External(format!("Failed to assign field ids: {e}")))?; - let schema = crate::models::Schema { - r#type: crate::models::schema::Type::Struct, - fields: schema.fields().to_vec(), - schema_id: Some(schema.schema_id()), - identifier_field_ids: Some(schema.identifier_field_ids().collect()), - }; + let schema = crate::r#gen::Schema::try_from(schema)?; - let sql_representation = crate::models::SqlViewRepresentation { + let sql_representation = crate::r#gen::SqlViewRepresentation { r#type: "sql".to_string(), sql: definition, dialect: "spark".to_string(), @@ -1451,14 +1494,16 @@ impl CatalogProvider for IcebergRestCatalogProvider { .map(|d| d.as_millis() as i64) .unwrap_or(0); - let view_version = crate::models::ViewVersion { + let view_version = crate::r#gen::ViewVersion { version_id: 1, // FIXME: When `replace` is supported and used, this should be a new version id. timestamp_ms, schema_id: -1, summary: HashMap::new(), - representations: vec![sql_representation], + representations: vec![crate::r#gen::ViewRepresentation::SqlViewRepresentation( + Box::new(sql_representation), + )], default_catalog: None, - default_namespace: database.clone().into(), + default_namespace: Box::new(database.clone().into()), }; let mut props = HashMap::new(); @@ -1479,13 +1524,13 @@ impl CatalogProvider for IcebergRestCatalogProvider { props.insert("comment".to_string(), c); } - if location.is_none() { - if let Some(path) = path { - props.insert("location".to_string(), path.clone()); - location = Some(path); - } + if location.is_none() + && let Some(path) = path + { + props.insert("location".to_string(), path.clone()); + location = Some(path); } - let request = crate::models::CreateViewRequest { + let request = crate::r#gen::CreateViewRequest { name: view.to_string(), location, schema: Box::new(schema), @@ -1493,64 +1538,92 @@ impl CatalogProvider for IcebergRestCatalogProvider { properties: props, }; - let result = client - .catalog_api_api() - .create_view( - &Self::namespace_string(catalog_config, database)?, - request, - Self::catalog_prefix(catalog_config), - ) - .await + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let request = request.clone(); + async move { client.create_view(prefix, namespace, request).await } + }) + .await? + .map(|response| response.inner) .map_err(|e| CatalogError::External(format!("Failed to create view: {e}")))?; - self.load_view_result_to_status(view, database, result) + Self::load_view_result_to_status(&self.name, database, view, result) } async fn get_view(&self, database: &Namespace, view: &str) -> CatalogResult { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - let result = client - .catalog_api_api() - .load_view( - &Self::namespace_string(catalog_config, database)?, - view, - Self::catalog_prefix(catalog_config), - ) - .await + let catalog_config = self.resolved_catalog_config().await?; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + let view_name = view.to_string(); + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let view_name = view_name.clone(); + async move { client.load_view(prefix, namespace, view_name).await } + }) + .await? + .map(|response| response.inner) .map_err(|e| match e { - apis::Error::ResponseError(apis::ResponseContent { status, .. }) - if status == 404 => + e if matches!( + e.status(), + Some(reqwest::StatusCode::METHOD_NOT_ALLOWED) + | Some(reqwest::StatusCode::NOT_IMPLEMENTED) + ) => { - CatalogError::NotFound( - CatalogObject::View, - format!( - "{}.{}", - quote_namespace_if_needed(database), - quote_name_if_needed(view) - ), - ) + CatalogError::NotSupported("get view".to_string()) } + e if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => CatalogError::NotFound( + CatalogObject::View, + format!( + "{}.{}", + quote_namespace_if_needed(database), + quote_name_if_needed(view) + ), + ), _ => CatalogError::External(format!( "Failed to load view {}.{}: {e}", quote_namespace_if_needed(database), quote_name_if_needed(view) )), })?; - self.load_view_result_to_status(view, database, result) + Self::load_view_result_to_status(&self.name, database, view, result) } async fn list_views(&self, database: &Namespace) -> CatalogResult> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; - - let result = client - .catalog_api_api() - .list_views( - &Self::namespace_string(catalog_config, database)?, - None, - None, - Self::catalog_prefix(catalog_config), - ) - .await - .map_err(|e| CatalogError::External(format!("Failed to list views: {e}")))?; + let catalog_config = self.resolved_catalog_config().await?; + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + + let result = self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + async move { client.list_views(prefix, namespace, None, None).await } + }) + .await? + .map(|response| response.inner) + .map_err(|e| match e { + e if matches!(e.status(), Some(reqwest::StatusCode::NOT_FOUND)) => { + CatalogError::NotFound( + CatalogObject::Namespace, + quote_namespace_if_needed(database), + ) + } + e if matches!( + e.status(), + Some(reqwest::StatusCode::METHOD_NOT_ALLOWED) + | Some(reqwest::StatusCode::NOT_IMPLEMENTED) + ) => + { + CatalogError::NotSupported("list views".to_string()) + } + _ => CatalogError::External(format!("Failed to list views: {e}")), + })?; let catalog = &self.name; Ok(result .identifiers @@ -1558,7 +1631,7 @@ impl CatalogProvider for IcebergRestCatalogProvider { .into_iter() .map(|identifier| TableStatus { catalog: Some(catalog.clone()), - database: identifier.namespace, + database: (*identifier.namespace).into(), name: identifier.name, kind: TableKind::View { definition: String::new(), @@ -1576,34 +1649,29 @@ impl CatalogProvider for IcebergRestCatalogProvider { view: &str, options: DropViewOptions, ) -> CatalogResult<()> { - let (client, catalog_config) = self.load_client_and_merged_config().await?; + let catalog_config = self.resolved_catalog_config().await?; let DropViewOptions { if_exists } = options; - match client - .catalog_api_api() - .drop_view( - &Self::namespace_string(catalog_config, database)?, - view, - Self::catalog_prefix(catalog_config), - ) - .await + let prefix = catalog_config.prefix().map(ToOwned::to_owned); + let namespace = catalog_config.namespace_string(database)?; + let view_name = view.to_string(); + match self + .with_auth_retry(|client| { + let prefix = prefix.clone(); + let namespace = namespace.clone(); + let view_name = view_name.clone(); + async move { client.drop_view(prefix, namespace, view_name).await } + }) + .await? { Ok(_) => Ok(()), - Err(apis::Error::ResponseError(apis::ResponseContent { status, .. })) - if status == 404 && if_exists => - { - Ok(()) - } + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) && if_exists => Ok(()), Err(e) => Err(CatalogError::External(format!("Failed to drop view: {e}"))), } } } /// Finds an item by ID, falling back to the last item if not found or no ID is provided. -fn find_by_id_or_last<'a, T, F>( - items: Option<&'a Vec>, - id: Option, - get_id: F, -) -> Option<&'a T> +fn find_by_id_or_last(items: Option<&Vec>, id: Option, get_id: F) -> Option<&T> where F: Fn(&T) -> Option, { @@ -1705,12 +1773,12 @@ fn columns_to_nested_fields( field = field.with_doc(comment); } - if format_version >= FormatVersion::V3 { - if let Some(default_literal) = default_literal { - field = field - .with_initial_default(default_literal.clone()) - .with_write_default(default_literal); - } + if format_version >= FormatVersion::V3 + && let Some(default_literal) = default_literal + { + field = field + .with_initial_default(default_literal.clone()) + .with_write_default(default_literal); } fields.push(Arc::new(field)); @@ -1730,7 +1798,7 @@ fn build_partition_spec( partition_by: &[CatalogPartitionField], bucket_by: Option<&CatalogTableBucketBy>, name_to_id: &HashMap, -) -> CatalogResult>> { +) -> CatalogResult>> { if partition_by.is_empty() && bucket_by.is_none() { return Ok(None); } @@ -1782,23 +1850,23 @@ fn build_partition_spec( if let Some(&source_id) = name_to_id.get(column) { partition_spec_builder = partition_spec_builder.add_field( source_id, - &format!("{column}_bucket"), + format!("{column}_bucket"), sail_iceberg::Transform::Bucket(num_buckets), ); } } } let spec = partition_spec_builder.build(); - Ok(Some(Box::new(crate::models::PartitionSpec { + Ok(Some(Box::new(crate::r#gen::PartitionSpec { spec_id: Some(spec.spec_id()), fields: spec .fields() .iter() - .map(|f| crate::models::PartitionField { + .map(|f| crate::r#gen::PartitionField { field_id: Some(f.field_id), source_id: f.source_id, name: f.name.to_string(), - transform: f.transform.to_string(), + transform: Box::new(crate::r#gen::Transform(f.transform.to_string())), }) .collect(), }))) @@ -1807,7 +1875,7 @@ fn build_partition_spec( fn build_sort_order( sort_by: &[CatalogTableSort], name_to_id: &HashMap, -) -> CatalogResult>> { +) -> CatalogResult>> { if sort_by.is_empty() { return Ok(None); } @@ -1839,25 +1907,25 @@ fn build_sort_order( fields: sort_fields, }; - Ok(Some(Box::new(crate::models::SortOrder { + Ok(Some(Box::new(crate::r#gen::SortOrder { order_id: i32::try_from(order.order_id).map_err(|e| { CatalogError::External(format!("Failed to convert sort order ID to i32: {e}")) })?, fields: order .fields .iter() - .map(|f| crate::models::SortField { + .map(|f| crate::r#gen::SortField { source_id: f.source_id, - transform: f.transform.to_string(), + transform: Box::new(crate::r#gen::Transform(f.transform.to_string())), direction: if f.direction == sail_iceberg::spec::sort::SortDirection::Ascending { - crate::models::SortDirection::Asc + Box::new(crate::r#gen::SortDirection::Asc) } else { - crate::models::SortDirection::Desc + Box::new(crate::r#gen::SortDirection::Desc) }, null_order: if f.null_order == sail_iceberg::spec::sort::NullOrder::First { - crate::models::NullOrder::NullsFirst + Box::new(crate::r#gen::NullOrder::NullsFirst) } else { - crate::models::NullOrder::NullsLast + Box::new(crate::r#gen::NullOrder::NullsLast) }, }) .collect(), @@ -1956,15 +2024,20 @@ fn parse_unary_sort_transform( #[expect(clippy::unwrap_used, clippy::panic)] #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use arrow::datatypes::DataType; + use sail_catalog::credentials::{EmptyCatalogCredentials, FileCatalogCredentials}; use sail_catalog::lakehouse::TableAccessPurpose; + use sail_common::spec; use sail_common_datafusion::catalog::{ CatalogProviderId, CatalogTableIdentity, CommitAuthority, LakehouseAuthority, LakehouseExecutionContext, LakehouseFormat, LakehouseOperation, MetadataPointerAuthority, TableLifecycle, }; + use tempfile::TempDir; use wiremock::matchers::{header, method, path, query_param, query_param_is_missing}; - use wiremock::{Mock, MockServer, ResponseTemplate}; + use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use super::*; @@ -1991,7 +2064,8 @@ mod tests { let name_str = name.unwrap_or(""); let props = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.uri())]); - let catalog = IcebergRestCatalogProvider::new(name_str.to_string(), props); + let catalog = + IcebergRestCatalogProvider::new(name_str.to_string(), test_options(props)); Self { name: name_str.to_string(), @@ -2043,6 +2117,13 @@ mod tests { } } + fn test_options(properties: HashMap) -> IcebergRestCatalogOptions { + IcebergRestCatalogOptions { + credentials: Arc::new(EmptyCatalogCredentials), + properties, + } + } + fn simple_create_table_options() -> CreateTableOptions { CreateTableOptions { columns: vec![CreateTableColumnOptions { @@ -2061,14 +2142,14 @@ mod tests { partition_by: vec![], sort_by: vec![], bucket_by: None, - mode: CreateTableMode::Create, + mode: spec::CreateTableMode::Create, properties: vec![], is_external: false, is_write_precondition: true, } } - fn create_table_response_with_access_session_requirements() -> serde_json::Value { + fn create_table_response_with_access_session_hints() -> serde_json::Value { serde_json::json!({ "metadata-location": "s3://bucket/table/metadata/v1.metadata.json", "metadata": { @@ -2092,7 +2173,6 @@ mod tests { ] }, "config": { - "scan-planning-mode": "server", "s3.remote-signing-enabled": "true" }, "storage-credentials": [ @@ -2107,12 +2187,18 @@ mod tests { }) } + fn create_table_response_with_server_side_scan_planning() -> serde_json::Value { + let mut result = create_table_response_with_access_session_hints(); + result["config"]["scan-planning-mode"] = serde_json::json!("server"); + result + } + async fn load_merged_test_config( defaults: HashMap, mut client_props: HashMap, overrides: HashMap, expected_config_warehouse: Option<&str>, - ) -> RestCatalogConfig { + ) -> CatalogConfig<'static> { let server = MockServer::start().await; let response = ResponseTemplate::new(200).set_body_json(serde_json::json!({ "defaults": defaults, @@ -2141,34 +2227,13 @@ mod tests { .entry(REST_CATALOG_PROP_URI.to_string()) .or_insert_with(|| server.uri()); - let catalog = IcebergRestCatalogProvider::new("test".to_string(), client_props); + let catalog = + IcebergRestCatalogProvider::new("test".to_string(), test_options(client_props)); - let (_, config) = catalog.load_client_and_merged_config().await.unwrap(); + let config = catalog.resolved_catalog_config().await.unwrap(); config.clone() } - #[test] - fn test_namespace_separator_aliases_are_canonicalized() { - for alias in [ - "namespace_separator", - "namespace-separator", - "namespaceSeparator", - "namespaceseparator", - ] { - let props = IcebergRestCatalogProvider::normalize_catalog_props(HashMap::from([( - alias.to_string(), - "::".to_string(), - )])); - - assert_eq!( - props - .get(REST_CATALOG_PROP_NAMESPACE_SEPARATOR) - .map(String::as_str), - Some("::") - ); - } - } - #[tokio::test] async fn test_catalog_config_merge_precedence_for_properties() { let key = "rest-page-size".to_string(); @@ -2180,7 +2245,10 @@ mod tests { None, ) .await; - assert_eq!(config.props.get(&key).map(String::as_str), Some("defaults")); + assert_eq!( + config.properties.get(&key).map(String::as_str), + Some("defaults") + ); let config = load_merged_test_config( HashMap::from([(key.clone(), "defaults".to_string())]), @@ -2189,7 +2257,10 @@ mod tests { None, ) .await; - assert_eq!(config.props.get(&key).map(String::as_str), Some("client")); + assert_eq!( + config.properties.get(&key).map(String::as_str), + Some("client") + ); let config = load_merged_test_config( HashMap::from([(key.clone(), "defaults".to_string())]), @@ -2199,7 +2270,7 @@ mod tests { ) .await; assert_eq!( - config.props.get(&key).map(String::as_str), + config.properties.get(&key).map(String::as_str), Some("overrides") ); } @@ -2216,7 +2287,7 @@ mod tests { None, ) .await; - assert_ne!(config.uri, "http://default.example"); + assert_ne!(config.uri().as_deref(), Some("http://default.example")); let config = load_merged_test_config( HashMap::new(), @@ -2228,14 +2299,14 @@ mod tests { None, ) .await; - assert_eq!(config.uri, "http://server.example"); + assert_eq!(config.uri().as_deref(), Some("http://server.example")); } #[tokio::test] async fn test_warehouse_merge_matrix() { let config = load_merged_test_config(HashMap::new(), HashMap::new(), HashMap::new(), None).await; - assert_eq!(config.warehouse, None); + assert_eq!(config.warehouse(), None); let config = load_merged_test_config( HashMap::from([( @@ -2247,7 +2318,10 @@ mod tests { None, ) .await; - assert_eq!(config.warehouse.as_deref(), Some("s3://default/warehouse")); + assert_eq!( + config.warehouse().as_deref(), + Some("s3://default/warehouse") + ); let config = load_merged_test_config( HashMap::from([( @@ -2262,7 +2336,7 @@ mod tests { Some("s3://client/warehouse"), ) .await; - assert_eq!(config.warehouse.as_deref(), Some("s3://client/warehouse")); + assert_eq!(config.warehouse().as_deref(), Some("s3://client/warehouse")); let config = load_merged_test_config( HashMap::from([( @@ -2280,14 +2354,14 @@ mod tests { Some("s3://client/warehouse"), ) .await; - assert_eq!(config.warehouse.as_deref(), Some("s3://server/warehouse")); + assert_eq!(config.warehouse().as_deref(), Some("s3://server/warehouse")); } #[tokio::test] async fn test_prefix_merge_matrix() { let config = load_merged_test_config(HashMap::new(), HashMap::new(), HashMap::new(), None).await; - assert_eq!(IcebergRestCatalogProvider::catalog_prefix(&config), None); + assert_eq!(config.prefix(), None); let config = load_merged_test_config( HashMap::from([( @@ -2299,10 +2373,7 @@ mod tests { None, ) .await; - assert_eq!( - IcebergRestCatalogProvider::catalog_prefix(&config), - Some("default_prefix") - ); + assert_eq!(config.prefix(), Some("default_prefix")); let config = load_merged_test_config( HashMap::from([( @@ -2317,10 +2388,7 @@ mod tests { None, ) .await; - assert_eq!( - IcebergRestCatalogProvider::catalog_prefix(&config), - Some("client_prefix") - ); + assert_eq!(config.prefix(), Some("client_prefix")); let config = load_merged_test_config( HashMap::from([( @@ -2338,10 +2406,7 @@ mod tests { None, ) .await; - assert_eq!( - IcebergRestCatalogProvider::catalog_prefix(&config), - Some("server_prefix") - ); + assert_eq!(config.prefix(), Some("server_prefix")); let config = load_merged_test_config( HashMap::new(), @@ -2353,7 +2418,7 @@ mod tests { None, ) .await; - assert_eq!(IcebergRestCatalogProvider::catalog_prefix(&config), None); + assert_eq!(config.prefix(), None); } #[tokio::test] @@ -2364,7 +2429,7 @@ mod tests { let config = load_merged_test_config(HashMap::new(), HashMap::new(), HashMap::new(), None).await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting\x1Ftax" ); @@ -2379,7 +2444,7 @@ mod tests { ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting/tax" ); @@ -2388,14 +2453,14 @@ mod tests { REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "/".to_string(), )]), - HashMap::from([("namespace_separator".to_string(), "%3A%3A".to_string())]), + HashMap::from([]), HashMap::new(), None, ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), - "accounting::tax" + config.namespace_string(&namespace).unwrap(), + "accounting/tax" ); let config = load_merged_test_config( @@ -2403,7 +2468,7 @@ mod tests { REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "/".to_string(), )]), - HashMap::from([("namespace_separator".to_string(), "::".to_string())]), + HashMap::from([]), HashMap::from([( REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "%7C".to_string(), @@ -2412,7 +2477,7 @@ mod tests { ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting|tax" ); @@ -2421,7 +2486,7 @@ mod tests { REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), "/".to_string(), )]), - HashMap::from([("namespace_separator".to_string(), "::".to_string())]), + HashMap::from([]), HashMap::from([( REST_CATALOG_PROP_NAMESPACE_SEPARATOR.to_string(), " ".to_string(), @@ -2430,7 +2495,7 @@ mod tests { ) .await; assert_eq!( - IcebergRestCatalogProvider::namespace_string(&config, &namespace).unwrap(), + config.namespace_string(&namespace).unwrap(), "accounting\x1Ftax" ); } @@ -2623,6 +2688,22 @@ mod tests { test_list_views_impl(Some("test")).await; } + #[tokio::test] + async fn test_list_views_unsupported_endpoint() { + let ctx = TestContext::new(None).await; + + Mock::given(method("GET")) + .and(path(ctx.path("/namespaces/ns1/views"))) + .respond_with(ResponseTemplate::new(405)) + .mount(&ctx.server) + .await; + + let namespace = Namespace::try_from(vec!["ns1".to_string()]).unwrap(); + let error = ctx.catalog.list_views(&namespace).await.unwrap_err(); + + assert!(matches!(error, CatalogError::NotSupported(_))); + } + async fn test_drop_database_impl(name: Option<&str>) { let ctx = TestContext::new(name).await; @@ -2969,12 +3050,16 @@ mod tests { _ => panic!("Expected PrimaryKey constraint"), } - assert!(properties - .iter() - .any(|(k, v)| k == "comment" && v == "test table")); - assert!(properties - .iter() - .any(|(k, v)| k == "owner" && v == "test_user")); + assert!( + properties + .iter() + .any(|(k, v)| k == "comment" && v == "test table") + ); + assert!( + properties + .iter() + .any(|(k, v)| k == "owner" && v == "test_user") + ); } _ => panic!("Expected Table kind"), } @@ -2987,13 +3072,33 @@ mod tests { } #[tokio::test] - async fn create_table_rejects_rest_access_session_requirements() { + async fn create_table_allows_rest_access_session_hints() { let ctx = TestContext::new(Some("test")).await; let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap(); ctx.mock_post_json( &ctx.path("/namespaces/db1/tables"), - create_table_response_with_access_session_requirements(), + create_table_response_with_access_session_hints(), + ) + .await; + + let status = ctx + .catalog + .create_table(&namespace, "table1", simple_create_table_options()) + .await + .unwrap(); + + assert_eq!(status.name, "table1"); + } + + #[tokio::test] + async fn create_table_rejects_server_side_scan_planning() { + let ctx = TestContext::new(Some("test")).await; + let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap(); + + ctx.mock_post_json( + &ctx.path("/namespaces/db1/tables"), + create_table_response_with_server_side_scan_planning(), ) .await; @@ -3004,17 +3109,17 @@ mod tests { .unwrap_err(); assert!(matches!(err, CatalogError::UnsupportedCapability(_))); - assert!(err.to_string().contains("Iceberg REST access session")); + assert!(err.to_string().contains("server-side scan planning")); } #[tokio::test] - async fn metadata_only_create_table_allows_rest_access_session_requirements() { + async fn metadata_only_create_table_allows_server_side_scan_planning() { let ctx = TestContext::new(Some("test")).await; let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap(); ctx.mock_post_json( &ctx.path("/namespaces/db1/tables"), - create_table_response_with_access_session_requirements(), + create_table_response_with_server_side_scan_planning(), ) .await; @@ -3224,12 +3329,16 @@ mod tests { assert_eq!(columns[1].comment, Some("filtered data".to_string())); assert_eq!(comment, Some("test view".to_string())); - assert!(properties - .iter() - .any(|(k, v)| k == "comment" && v == "test view")); - assert!(properties - .iter() - .any(|(k, v)| k == "created_by" && v == "test_user")); + assert!( + properties + .iter() + .any(|(k, v)| k == "comment" && v == "test view") + ); + assert!( + properties + .iter() + .any(|(k, v)| k == "created_by" && v == "test_user") + ); } _ => panic!("Expected View kind"), } @@ -3241,6 +3350,22 @@ mod tests { test_get_view_impl(Some("test")).await; } + #[tokio::test] + async fn test_get_view_unsupported_endpoint() { + let ctx = TestContext::new(None).await; + + Mock::given(method("GET")) + .and(path(ctx.path("/namespaces/db1/views/view1"))) + .respond_with(ResponseTemplate::new(501)) + .mount(&ctx.server) + .await; + + let namespace = Namespace::try_from(vec!["db1".to_string()]).unwrap(); + let error = ctx.catalog.get_view(&namespace, "view1").await.unwrap_err(); + + assert!(matches!(error, CatalogError::NotSupported(_))); + } + async fn test_create_database_impl(name: Option<&str>) { let ctx = TestContext::new(name).await; @@ -3285,10 +3410,11 @@ mod tests { assert_eq!(db.database, vec!["db1".to_string()]); assert_eq!(db.comment, Some("test database".to_string())); assert_eq!(db.location, Some("s3://bucket/db1".to_string())); - assert!(db - .properties - .iter() - .any(|(k, v)| k == "custom_prop" && v == "custom_value")); + assert!( + db.properties + .iter() + .any(|(k, v)| k == "custom_prop" && v == "custom_value") + ); Mock::given(method("POST")) .and(path(ctx.path("/namespaces").as_str())) @@ -3377,10 +3503,11 @@ mod tests { assert_eq!(db.database, vec!["db1".to_string()]); assert_eq!(db.comment, Some("test database".to_string())); assert_eq!(db.location, Some("s3://bucket/db1".to_string())); - assert!(db - .properties - .iter() - .any(|(k, v)| k == "custom_prop" && v == "custom_value")); + assert!( + db.properties + .iter() + .any(|(k, v)| k == "custom_prop" && v == "custom_value") + ); } #[tokio::test] @@ -3412,22 +3539,30 @@ mod tests { assert_eq!(result.database, vec!["db1".to_string()]); assert_eq!(result.comment, Some("test database".to_string())); assert_eq!(result.location, Some("s3://bucket/db1".to_string())); - assert!(result - .properties - .iter() - .any(|(k, v)| k == "comment" && v == "test database")); - assert!(result - .properties - .iter() - .any(|(k, v)| k == "location" && v == "s3://bucket/db1")); - assert!(result - .properties - .iter() - .any(|(k, v)| k == "owner" && v == "alice")); - assert!(result - .properties - .iter() - .any(|(k, v)| k == "custom_prop" && v == "custom_value")); + assert!( + result + .properties + .iter() + .any(|(k, v)| k == "comment" && v == "test database") + ); + assert!( + result + .properties + .iter() + .any(|(k, v)| k == "location" && v == "s3://bucket/db1") + ); + assert!( + result + .properties + .iter() + .any(|(k, v)| k == "owner" && v == "alice") + ); + assert!( + result + .properties + .iter() + .any(|(k, v)| k == "custom_prop" && v == "custom_value") + ); ctx.mock_get_json( &ctx.path("/namespaces/db2"), @@ -3471,4 +3606,225 @@ mod tests { test_get_database_impl(None).await; test_get_database_impl(Some("test")).await; } + + fn error_with_status(status: reqwest::StatusCode) -> ApiError<()> { + ApiError::Unknown(crate::r#gen::Response { + inner: (), + status, + headers: reqwest::header::HeaderMap::new(), + }) + } + + #[tokio::test] + async fn bootstrap_config_recovers_when_token_rotates_before_response() { + let dir = TempDir::new().unwrap(); + let token_path = dir.path().join("token"); + std::fs::write(&token_path, "token-a").unwrap(); + + let server = MockServer::start().await; + let rotate_path = token_path.clone(); + Mock::given(method("GET")) + .and(path("/v1/config")) + .and(header("authorization", "Bearer token-a")) + .respond_with(move |_req: &Request| { + std::fs::write(&rotate_path, "token-b").unwrap(); + ResponseTemplate::new(401).set_body_json(serde_json::json!({ + "error": { + "message": "token expired", + "type": "NotAuthorizedException", + "code": 401 + } + })) + }) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/config")) + .and(header("authorization", "Bearer token-b")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "overrides": { "warehouse": "s3://iceberg-catalog" }, + "defaults": {} + }))) + .expect(1) + .mount(&server) + .await; + + let properties = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.uri())]); + let catalog = IcebergRestCatalogProvider::new( + String::new(), + IcebergRestCatalogOptions { + credentials: Arc::new(FileCatalogCredentials::new(&token_path)), + properties, + }, + ); + + let config = catalog.resolved_catalog_config().await.unwrap(); + assert_eq!(config.warehouse().as_deref(), Some("s3://iceberg-catalog")); + assert_eq!(std::fs::read_to_string(&token_path).unwrap(), "token-b"); + } + + #[tokio::test] + async fn with_auth_retry_retries_once_on_unauthorized() { + let ctx = TestContext::new(None).await; + let calls = AtomicUsize::new(0); + let outcome: Result<(), ApiError<()>> = ctx + .catalog + .with_auth_retry(|_client| { + let attempt = calls.fetch_add(1, Ordering::SeqCst); + async move { + if attempt == 0 { + Err(error_with_status(reqwest::StatusCode::UNAUTHORIZED)) + } else { + Ok(()) + } + } + }) + .await + .unwrap(); + assert!(outcome.is_ok()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn with_auth_retry_does_not_retry_more_than_once() { + let ctx = TestContext::new(None).await; + let calls = AtomicUsize::new(0); + let outcome: Result<(), ApiError<()>> = ctx + .catalog + .with_auth_retry(|_client| { + calls.fetch_add(1, Ordering::SeqCst); + async move { Err(error_with_status(reqwest::StatusCode::UNAUTHORIZED)) } + }) + .await + .unwrap(); + assert!(outcome.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn with_auth_retry_does_not_retry_non_unauthorized() { + let ctx = TestContext::new(None).await; + let calls = AtomicUsize::new(0); + let outcome: Result<(), ApiError<()>> = ctx + .catalog + .with_auth_retry(|_client| { + calls.fetch_add(1, Ordering::SeqCst); + async move { + Err(error_with_status( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + )) + } + }) + .await + .unwrap(); + assert!(outcome.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn drop_database_cascade_recovers_when_token_rotates_midway() { + let dir = TempDir::new().unwrap(); + let token_path = dir.path().join("token"); + std::fs::write(&token_path, "token-a").unwrap(); + + let server = MockServer::start().await; + + // Bootstrap config. Reachable with the original token. + Mock::given(method("GET")) + .and(path("/v1/config")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "overrides": { "warehouse": "s3://iceberg-catalog" }, + "defaults": {} + }))) + .mount(&server) + .await; + + // The namespace still holds one table when the cascade begins. Listing + // is authorized with the original token. + Mock::given(method("GET")) + .and(path("/v1/namespaces/dbc/tables")) + .and(header("authorization", "Bearer token-a")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "identifiers": [ { "namespace": ["dbc"], "name": "t1" } ] + }))) + .mount(&server) + .await; + + // The first drop of the table arrives with the old token. The server + // rejects it with 401 and, at that moment, the projected token file + // rotates to a new value (as kubelet would swap it). + let rotate_path = token_path.clone(); + Mock::given(method("DELETE")) + .and(path("/v1/namespaces/dbc/tables/t1")) + .and(header("authorization", "Bearer token-a")) + .respond_with(move |_req: &Request| { + std::fs::write(&rotate_path, "token-b").unwrap(); + ResponseTemplate::new(401).set_body_json(serde_json::json!({ + "error": { + "message": "token expired", + "type": "NotAuthorizedException", + "code": 401 + } + })) + }) + .expect(1) + .mount(&server) + .await; + + // The retry re-reads the rotated token and is authorized. + Mock::given(method("DELETE")) + .and(path("/v1/namespaces/dbc/tables/t1")) + .and(header("authorization", "Bearer token-b")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + + // The remaining cascade requests all use the rotated token. + Mock::given(method("GET")) + .and(path("/v1/namespaces/dbc/views")) + .and(header("authorization", "Bearer token-b")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "identifiers": [] + }))) + .expect(1) + .mount(&server) + .await; + + Mock::given(method("DELETE")) + .and(path("/v1/namespaces/dbc")) + .and(header("authorization", "Bearer token-b")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + + let props = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.uri())]); + let options = IcebergRestCatalogOptions { + credentials: Arc::new(FileCatalogCredentials::new(&token_path)), + properties: props, + }; + let catalog = IcebergRestCatalogProvider::new(String::new(), options); + + let namespace = Namespace::try_from(vec!["dbc".to_string()]).unwrap(); + let result = catalog + .drop_database( + &namespace, + DropDatabaseOptions { + if_exists: false, + cascade: true, + }, + ) + .await; + + assert!(result.is_ok(), "cascade drop should succeed: {result:?}"); + // The rotated token is the one the file ends up holding, and every + // mounted request expectation (including the retried table drop) is + // verified when the server is dropped. + assert_eq!( + std::fs::read_to_string(&token_path).unwrap(), + "token-b".to_string() + ); + } } diff --git a/crates/sail-catalog-iceberg/tests/rest_integration_test.rs b/crates/sail-catalog-iceberg/tests/rest_integration_test.rs index 1219f8dbf..15ca79f3f 100644 --- a/crates/sail-catalog-iceberg/tests/rest_integration_test.rs +++ b/crates/sail-catalog-iceberg/tests/rest_integration_test.rs @@ -13,15 +13,19 @@ #![expect(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::collections::HashMap; +use std::sync::Arc; use arrow::datatypes::DataType; +use sail_catalog::credentials::EmptyCatalogCredentials; use sail_catalog::provider::{ CatalogPartitionField, CatalogProvider, CreateDatabaseOptions, CreateTableColumnOptions, CreateTableMode, CreateTableOptions, CreateViewColumnOptions, CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, PartitionTransform, RuntimeAwareCatalogProvider, }; -use sail_catalog_iceberg::{IcebergRestCatalogProvider, REST_CATALOG_PROP_URI}; +use sail_catalog_iceberg::{ + IcebergRestCatalogOptions, IcebergRestCatalogProvider, REST_CATALOG_PROP_URI, +}; use sail_common::runtime::RuntimeHandle; use sail_common_datafusion::catalog::{ CatalogTableConstraint, CatalogTableSort, TableColumnStatus, TableKind, @@ -108,7 +112,13 @@ async fn setup_catalog( let catalog = RuntimeAwareCatalogProvider::try_new( || { let props = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), rest_url)]); - let provider = IcebergRestCatalogProvider::new("test".to_string(), props); + let provider = IcebergRestCatalogProvider::new( + "test".to_string(), + IcebergRestCatalogOptions { + credentials: Arc::new(EmptyCatalogCredentials), + properties: props, + }, + ); Ok(provider) }, runtime.io().clone(), @@ -141,10 +151,12 @@ async fn test_create_namespace() { assert_eq!(created_db.database, Vec::::from(namespace.clone())); assert_eq!(created_db.comment, Some("test comment".to_string())); assert_eq!(created_db.location, Some("s3://bucket/path".to_string())); - assert!(created_db - .properties - .iter() - .any(|(k, v)| k == "key1" && v == "value1")); + assert!( + created_db + .properties + .iter() + .any(|(k, v)| k == "key1" && v == "value1") + ); let result = rest_catalog .create_database( @@ -248,19 +260,23 @@ async fn test_get_namespace() { assert_eq!(created_db.database, Vec::::from(namespace.clone())); for (key, value) in &properties { - assert!(created_db - .properties - .iter() - .any(|(k, v)| k == key && v == value)); + assert!( + created_db + .properties + .iter() + .any(|(k, v)| k == key && v == value) + ); } let get_db = rest_catalog.get_database(&namespace).await.unwrap(); assert_eq!(get_db.database, Vec::::from(namespace)); for (key, value) in &properties { - assert!(get_db - .properties - .iter() - .any(|(k, v)| k == key && v == value)); + assert!( + get_db + .properties + .iter() + .any(|(k, v)| k == key && v == value) + ); } } @@ -316,12 +332,14 @@ async fn test_list_namespaces() { let dbs = rest_catalog.list_databases(Some(&parent)).await.unwrap(); assert_eq!(dbs.len(), 2); - assert!(dbs - .iter() - .any(|db| db.database == Vec::::from(ns1.clone()))); - assert!(dbs - .iter() - .any(|db| db.database == Vec::::from(ns2.clone()))); + assert!( + dbs.iter() + .any(|db| db.database == Vec::::from(ns1.clone())) + ); + assert!( + dbs.iter() + .any(|db| db.database == Vec::::from(ns2.clone())) + ); } #[tokio::test] @@ -661,12 +679,16 @@ async fn test_create_table() { assert_eq!(static_properties, expected_properties); assert!(properties.iter().any(|(k, v)| k == "metadata-location" && v.starts_with("s3://icebergdata/demo/test_create_table/apple/ios/t1/metadata/"))); - assert!(properties - .iter() - .any(|(k, v)| k == "metadata.last-updated-ms" && !v.is_empty())); - assert!(properties - .iter() - .any(|(k, v)| k == "metadata.table-uuid" && !v.is_empty())); + assert!( + properties + .iter() + .any(|(k, v)| k == "metadata.last-updated-ms" && !v.is_empty()) + ); + assert!( + properties + .iter() + .any(|(k, v)| k == "metadata.table-uuid" && !v.is_empty()) + ); assert_eq!(table.name, "t1".to_string()); assert_eq!(table.catalog, Some("test".to_string())); @@ -1069,12 +1091,16 @@ async fn test_get_table() { assert_eq!(static_properties, expected_properties); assert!(properties.iter().any(|(k, v)| k == "metadata-location" && v.starts_with("s3://icebergdata/custom/path/meow/metadata/"))); - assert!(properties - .iter() - .any(|(k, v)| k == "metadata.last-updated-ms" && !v.is_empty())); - assert!(properties - .iter() - .any(|(k, v)| k == "metadata.table-uuid" && !v.is_empty())); + assert!( + properties + .iter() + .any(|(k, v)| k == "metadata.last-updated-ms" && !v.is_empty()) + ); + assert!( + properties + .iter() + .any(|(k, v)| k == "metadata.table-uuid" && !v.is_empty()) + ); assert_eq!(table.name, "t2".to_string()); assert_eq!(table.catalog, Some("test".to_string())); @@ -1460,9 +1486,11 @@ async fn test_create_view() { assert_eq!(properties.len(), 6); assert!(properties.iter().any(|(k, v)| k == "metadata-location" && v.starts_with("s3://icebergdata/demo/test_create_view/view1/metadata/"))); - assert!(properties - .iter() - .any(|(k, v)| k == "metadata.view-uuid" && !v.is_empty())); + assert!( + properties + .iter() + .any(|(k, v)| k == "metadata.view-uuid" && !v.is_empty()) + ); assert_eq!(static_properties, expected_properties); assert_eq!(view.name, "view1".to_string()); diff --git a/crates/sail-catalog-memory/src/provider.rs b/crates/sail-catalog-memory/src/provider.rs index b9331bcb4..d7b8bcbf5 100644 --- a/crates/sail-catalog-memory/src/provider.rs +++ b/crates/sail-catalog-memory/src/provider.rs @@ -3,15 +3,15 @@ use std::collections::HashMap; use dashmap::{DashMap, Entry}; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::provider::{ - plain_lakehouse_create_table_metadata_requirement, AlterTableOptions, CatalogProvider, - CreateDatabaseOptions, CreateTableColumnOptions, CreateTableMetadataRequirement, - CreateTableMode, CreateTableOptions, CreateViewColumnOptions, CreateViewOptions, - DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, + AlterTableOptions, CatalogProvider, CreateDatabaseOptions, CreateTableColumnOptions, + CreateTableMetadataRequirement, CreateTableMode, CreateTableOptions, CreateViewColumnOptions, + CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, + plain_lakehouse_create_table_metadata_requirement, }; use sail_catalog::utils::quote_namespace_if_needed; use sail_common_datafusion::catalog::{ - alter_column_default, alter_column_type, CatalogPartitionField, DatabaseStatus, - TableColumnStatus, TableKind, TableStatus, + CatalogPartitionField, DatabaseStatus, TableColumnStatus, TableKind, TableStatus, + alter_column_default, alter_column_type, }; use crate::managed_table; @@ -115,13 +115,12 @@ impl CatalogProvider for MemoryCatalogProvider { } async fn get_database(&self, database: &Namespace) -> CatalogResult { - if let Some(db) = self.databases.get(database) { - Ok(db.status.clone()) - } else { - Err(CatalogError::NotFound( + match self.databases.get(database) { + Some(db) => Ok(db.status.clone()), + _ => Err(CatalogError::NotFound( CatalogObject::Database, quote_namespace_if_needed(database), - )) + )), } } @@ -148,21 +147,28 @@ impl CatalogProvider for MemoryCatalogProvider { database: &Namespace, options: DropDatabaseOptions, ) -> CatalogResult<()> { - let DropDatabaseOptions { - if_exists, - cascade: _, - } = options; - if self.databases.remove(database).is_none() { - if if_exists { + let DropDatabaseOptions { if_exists, cascade } = options; + match self.databases.entry(database.clone()) { + Entry::Occupied(entry) => { + if !cascade && (!entry.get().tables.is_empty() || !entry.get().views.is_empty()) { + return Err(CatalogError::InvalidArgument(format!( + "Cannot drop non-empty database '{}', use CASCADE", + quote_namespace_if_needed(database), + ))); + } + let _ = entry.remove_entry(); Ok(()) - } else { - Err(CatalogError::NotFound( - CatalogObject::Database, - quote_namespace_if_needed(database), - )) } - } else { - Ok(()) + Entry::Vacant(_) => { + if if_exists { + Ok(()) + } else { + Err(CatalogError::NotFound( + CatalogObject::Database, + quote_namespace_if_needed(database), + )) + } + } } } @@ -271,10 +277,10 @@ impl CatalogProvider for MemoryCatalogProvider { } async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { - if let Some(db) = self.databases.get(database) { - if let Some(status) = db.tables.get(table) { - return Ok(status.clone()); - } + if let Some(db) = self.databases.get(database) + && let Some(status) = db.tables.get(table) + { + return Ok(status.clone()); } Err(CatalogError::NotFound( CatalogObject::Table, @@ -283,13 +289,12 @@ impl CatalogProvider for MemoryCatalogProvider { } async fn list_tables(&self, database: &Namespace) -> CatalogResult> { - if let Some(db) = self.databases.get(database) { - Ok(db.tables.values().cloned().collect()) - } else { - Err(CatalogError::NotFound( + match self.databases.get(database) { + Some(db) => Ok(db.tables.values().cloned().collect()), + _ => Err(CatalogError::NotFound( CatalogObject::Database, quote_namespace_if_needed(database), - )) + )), } } @@ -306,22 +311,27 @@ impl CatalogProvider for MemoryCatalogProvider { if_exists, purge: _, } = options; - if let Some(mut db) = self.databases.get_mut(database) { - if db.tables.remove(table).is_some() || if_exists { - Ok(()) - } else { - Err(CatalogError::NotFound( - CatalogObject::Table, - table.to_string(), - )) + match self.databases.get_mut(database) { + Some(mut db) => { + if db.tables.remove(table).is_some() || if_exists { + Ok(()) + } else { + Err(CatalogError::NotFound( + CatalogObject::Table, + table.to_string(), + )) + } + } + _ => { + if if_exists { + Ok(()) + } else { + Err(CatalogError::NotFound( + CatalogObject::Database, + quote_namespace_if_needed(database), + )) + } } - } else if if_exists { - Ok(()) - } else { - Err(CatalogError::NotFound( - CatalogObject::Database, - quote_namespace_if_needed(database), - )) } } @@ -466,10 +476,10 @@ impl CatalogProvider for MemoryCatalogProvider { } async fn get_view(&self, database: &Namespace, view: &str) -> CatalogResult { - if let Some(db) = self.databases.get(database) { - if let Some(status) = db.views.get(view) { - return Ok(status.clone()); - } + if let Some(db) = self.databases.get(database) + && let Some(status) = db.views.get(view) + { + return Ok(status.clone()); } Err(CatalogError::NotFound( CatalogObject::View, @@ -478,13 +488,12 @@ impl CatalogProvider for MemoryCatalogProvider { } async fn list_views(&self, database: &Namespace) -> CatalogResult> { - if let Some(db) = self.databases.get(database) { - Ok(db.views.values().cloned().collect()) - } else { - Err(CatalogError::NotFound( + match self.databases.get(database) { + Some(db) => Ok(db.views.values().cloned().collect()), + _ => Err(CatalogError::NotFound( CatalogObject::Database, quote_namespace_if_needed(database), - )) + )), } } @@ -495,22 +504,27 @@ impl CatalogProvider for MemoryCatalogProvider { options: DropViewOptions, ) -> CatalogResult<()> { let DropViewOptions { if_exists } = options; - if let Some(mut db) = self.databases.get_mut(database) { - if db.views.remove(view).is_some() || if_exists { - Ok(()) - } else { - Err(CatalogError::NotFound( - CatalogObject::View, - view.to_string(), - )) + match self.databases.get_mut(database) { + Some(mut db) => { + if db.views.remove(view).is_some() || if_exists { + Ok(()) + } else { + Err(CatalogError::NotFound( + CatalogObject::View, + view.to_string(), + )) + } + } + _ => { + if if_exists { + Ok(()) + } else { + Err(CatalogError::NotFound( + CatalogObject::Database, + quote_namespace_if_needed(database), + )) + } } - } else if if_exists { - Ok(()) - } else { - Err(CatalogError::NotFound( - CatalogObject::Database, - quote_namespace_if_needed(database), - )) } } } diff --git a/crates/sail-catalog-onelake/Cargo.toml b/crates/sail-catalog-onelake/Cargo.toml index fab8543d7..41c1d24f0 100644 --- a/crates/sail-catalog-onelake/Cargo.toml +++ b/crates/sail-catalog-onelake/Cargo.toml @@ -9,9 +9,10 @@ workspace = true [dependencies] sail-common-datafusion = { path = "../sail-common-datafusion" } sail-catalog = { path = "../sail-catalog" } +sail-catalog-iceberg = { path = "../sail-catalog-iceberg" } +sail-catalog-unity = { path = "../sail-catalog-unity" } -arrow = { workspace = true } async-trait = { workspace = true } -reqwest = { workspace = true } -serde = { workspace = true } +object_store = { workspace = true } tokio = { workspace = true } +url = { workspace = true } diff --git a/crates/sail-catalog-onelake/src/credentials.rs b/crates/sail-catalog-onelake/src/credentials.rs new file mode 100644 index 000000000..ac8a114c0 --- /dev/null +++ b/crates/sail-catalog-onelake/src/credentials.rs @@ -0,0 +1,63 @@ +use object_store::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzureBuilder}; +use sail_catalog::credentials::CatalogCredentials; +use sail_catalog::error::{CatalogError, CatalogResult}; +use tokio::sync::OnceCell; + +#[derive(Debug)] +pub enum OneLakeCredentials { + Static { + bearer_token: String, + }, + Dynamic { + workspace: String, + provider: OnceCell, + }, +} + +#[async_trait::async_trait] +impl CatalogCredentials for OneLakeCredentials { + async fn retrieve(&self) -> CatalogResult> { + match self { + Self::Static { bearer_token } => Ok(Some(bearer_token.clone())), + Self::Dynamic { + workspace, + provider, + } => { + let provider = provider + .get_or_try_init(|| async { + let mut builder = MicrosoftAzureBuilder::from_env() + .with_account("onelake") + .with_container_name(workspace) + .with_use_fabric_endpoint(true); + + if std::env::var("AZURE_STORAGE_TOKEN").is_err() + && let Ok(token) = std::env::var("AZURE_ACCESS_TOKEN") + { + builder = builder.with_bearer_token_authorization(token); + } + + let store = builder.build().map_err(|e| { + CatalogError::External(format!( + "Failed to build OneLake Azure credential provider: {e}" + )) + })?; + Ok::(store.credentials().clone()) + }) + .await?; + + let credential = provider.get_credential().await.map_err(|e| { + CatalogError::External(format!("Failed to get OneLake token: {e}")) + })?; + + match credential.as_ref() { + AzureCredential::BearerToken(token) => Ok(Some(token.clone())), + AzureCredential::AccessKey(_) | AzureCredential::SASToken(_) => { + Err(CatalogError::External( + "OneLake catalog requires Microsoft Entra bearer token authentication; Azure access keys and SAS tokens are not supported".to_string(), + )) + } + } + } + } + } +} diff --git a/crates/sail-catalog-onelake/src/lib.rs b/crates/sail-catalog-onelake/src/lib.rs index 9e0a63a22..ecf64b683 100644 --- a/crates/sail-catalog-onelake/src/lib.rs +++ b/crates/sail-catalog-onelake/src/lib.rs @@ -1,3 +1,5 @@ +mod credentials; mod provider; -pub use provider::OneLakeCatalogProvider; +pub use credentials::OneLakeCredentials; +pub use provider::{OneLakeApiKind, OneLakeCatalogProvider}; diff --git a/crates/sail-catalog-onelake/src/provider.rs b/crates/sail-catalog-onelake/src/provider.rs index ede79df10..5f60d3e07 100644 --- a/crates/sail-catalog-onelake/src/provider.rs +++ b/crates/sail-catalog-onelake/src/provider.rs @@ -1,133 +1,57 @@ +use std::collections::HashMap; use std::sync::Arc; -use reqwest::Client; -use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; +use sail_catalog::credentials::CatalogCredentials; +use sail_catalog::error::CatalogResult; +use sail_catalog::lakehouse::{ + BeginTableAccessRequest, DeltaRatifiedCommitRequest, DeltaRatifiedCommitResponse, + LakehouseCapability, LakehouseCommitOutcome, LakehouseCommitRequest, LakehouseCreatePlan, + LakehouseCreateRequest, LakehouseResolvedTable, LakehouseScanPlanningRequest, + LakehouseScanPlanningResponse, ResolveLakehouseTableRequest, TableAccessSession, +}; use sail_catalog::provider::{ - AlterTableOptions, CatalogProvider, CreateDatabaseOptions, CreateTableOptions, - CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, + AlterTableOptions, CatalogProvider, CreateDatabaseOptions, CreateTableMetadataRequirement, + CreateTableOptions, CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, + Namespace, +}; +use sail_catalog_iceberg::{ + IcebergRestCatalogOptions, IcebergRestCatalogProvider, REST_CATALOG_PROP_PREFIX, + REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, }; -use sail_catalog::utils::quote_namespace_if_needed; -use sail_common_datafusion::catalog::{DatabaseStatus, TableColumnStatus, TableKind, TableStatus}; -use serde::Deserialize; +use sail_catalog_unity::{UnityCatalogOptions, UnityCatalogProvider}; +use sail_common_datafusion::catalog::{DatabaseStatus, TableKind, TableStatus}; use tokio::sync::OnceCell; +use url::Url; -const ONELAKE_TABLE_API_BASE: &str = "https://onelake.table.fabric.microsoft.com/delta"; - -/// Fetches Azure access token using Azure CLI -async fn get_azure_cli_token() -> CatalogResult { - // On Windows, az is a batch file (.cmd) and must be run through cmd.exe - // See: https://doc.rust-lang.org/nightly/std/process/struct.Command.html - let (program, extra_args): (&str, &[&str]) = if cfg!(target_os = "windows") { - ("cmd", &["/C", "az"]) - } else { - ("az", &[]) - }; - - let output = tokio::process::Command::new(program) - .args(extra_args) - .args([ - "account", - "get-access-token", - "--resource", - "https://storage.azure.com/", - "--query", - "accessToken", - "-o", - "tsv", - ]) - .output() - .await - .map_err(|e| CatalogError::External(format!("Failed to run az cli: {e}")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(CatalogError::External(format!( - "Azure CLI failed: {stderr}" - ))); - } - - let token = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if token.is_empty() { - return Err(CatalogError::External( - "Azure CLI returned empty token".to_string(), - )); - } - - Ok(token) -} +use crate::credentials::OneLakeCredentials; -/// Converts OneLake https:// URL to abfss:// URL for Delta Lake access -/// From: https://onelake.dfs.fabric.microsoft.com/workspace/item.ItemType/Tables/schema/table -/// To: abfss://workspace@onelake.dfs.fabric.microsoft.com/item.ItemType/Tables/schema/table -fn convert_to_abfss_url(https_url: &str) -> Option { - let url = https_url.strip_prefix("https://onelake.dfs.fabric.microsoft.com/")?; - let (workspace, rest) = url.split_once('/')?; - Some(format!( - "abfss://{}@onelake.dfs.fabric.microsoft.com/{}", - workspace, rest - )) +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum OneLakeApiKind { + Delta, + Iceberg, } -/// Response from list schemas API -#[derive(Debug, Deserialize)] -struct ListSchemasResponse { - schemas: Vec, -} +const ONE_LAKE_DELTA_ENDPOINT: &str = "https://onelake.table.fabric.microsoft.com/delta"; +const ONE_LAKE_ICEBERG_ENDPOINT: &str = "https://onelake.table.fabric.microsoft.com/iceberg"; -#[derive(Debug, Deserialize)] -struct SchemaInfo { - name: Option, - #[expect(dead_code)] - catalog_name: Option, - comment: Option, -} - -/// Response from list tables API -#[derive(Debug, Deserialize)] -struct ListTablesResponse { - tables: Vec, -} - -#[derive(Debug, Deserialize)] -struct TableInfo { - name: Option, - #[expect(dead_code)] - catalog_name: Option, - schema_name: Option, - #[expect(dead_code)] - table_type: Option, - data_source_format: Option, - storage_location: Option, - columns: Option>, - comment: Option, -} - -#[derive(Debug, Deserialize)] -struct ColumnInfo { - name: Option, - type_name: Option, - type_text: Option, - nullable: Option, - comment: Option, -} - -/// Configuration for OneLake catalog +/// Configuration for OneLake catalog. #[derive(Debug, Clone)] pub struct OneLakeCatalogConfig { + #[expect(unused)] pub workspace: String, pub item_name: String, - pub item_type: String, - pub bearer_token: Option, + pub item_type: Option, + pub api: OneLakeApiKind, } /// Provider for Microsoft Fabric OneLake catalog. /// -/// This catalog uses the OneLake Table APIs to list schemas and tables, -/// and returns Delta table locations that can be read/written via abfss://. +/// This catalog uses the OneLake Table APIs and delegates protocol-specific catalog +/// behavior to the Iceberg REST or Unity Catalog providers. pub struct OneLakeCatalogProvider { name: String, config: OneLakeCatalogConfig, - client: OnceCell>, + inner: Arc, } impl OneLakeCatalogProvider { @@ -135,191 +59,169 @@ impl OneLakeCatalogProvider { name: String, workspace: String, item_name: String, - item_type: String, + item_type: Option, + api: OneLakeApiKind, bearer_token: Option, - ) -> Self { - Self { + ) -> CatalogResult { + let credentials = match bearer_token { + Some(bearer_token) => OneLakeCredentials::Static { bearer_token }, + None => OneLakeCredentials::Dynamic { + workspace: workspace.clone(), + provider: OnceCell::new(), + }, + }; + let credentials = Arc::new(credentials) as Arc; + // `/.` for friendly names, or + // `/`(no type segment) for GUIDs + let item_path = onelake_item_path(&workspace, &item_name, item_type.as_deref()); + let inner: Arc = match api { + OneLakeApiKind::Delta => { + let uri = format!( + "{}/{}/api/2.1/unity-catalog", + ONE_LAKE_DELTA_ENDPOINT, item_path + ); + let catalog_name = onelake_catalog_name(&item_name, item_type.as_deref()); + Arc::new(UnityCatalogProvider::new( + name.clone(), + UnityCatalogOptions { + default_catalog: catalog_name, + uri, + credentials, + quote_object_name: false, + }, + )?) + } + OneLakeApiKind::Iceberg => { + let uri = ONE_LAKE_ICEBERG_ENDPOINT.to_string(); + let mut properties = HashMap::new(); + properties.insert(REST_CATALOG_PROP_URI.to_string(), uri); + properties.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), item_path.clone()); + properties.insert(REST_CATALOG_PROP_PREFIX.to_string(), item_path); + Arc::new(IcebergRestCatalogProvider::new( + name.clone(), + IcebergRestCatalogOptions { + credentials, + properties, + }, + )) + } + }; + + Ok(Self { name, config: OneLakeCatalogConfig { workspace, item_name, item_type, - bearer_token, + api, }, - client: OnceCell::new(), - } + inner, + }) } - fn base_url(&self) -> String { - format!( - "{}/{}/{}.{}", - ONELAKE_TABLE_API_BASE, - self.config.workspace, - self.config.item_name, - self.config.item_type - ) + fn catalog_name(&self) -> String { + onelake_catalog_name(&self.config.item_name, self.config.item_type.as_deref()) } - fn schema_name(database: &Namespace) -> CatalogResult { - if database.tail.is_empty() { - Ok(database.head.to_string()) - } else { - Err(CatalogError::InvalidArgument(format!( - "OneLake catalog does not support multi-level namespaces: {}", - quote_namespace_if_needed(database) - ))) + fn normalize_database_status(&self, mut status: DatabaseStatus) -> DatabaseStatus { + if matches!(self.config.api, OneLakeApiKind::Delta) { + status.database = self.normalize_delta_database(status.database); + } + // OneLake's Iceberg REST API returns the namespace `location` as a workspace-relative + // path (e.g. `workspace/item.Type/Tables/schema`). `resolve_default_table_location` then + // seems to append the table name and hand the result to the Iceberg writer, which seems + // to reject any non-absolute location, so convert it to an absolute `abfss://` URL. + if let Some(location) = status.location.take() { + status.location = Some(normalize_onelake_location(&location).unwrap_or(location)); } + // Keep the mirrored `location` table property consistent with the field above + // (DESCRIBE DATABASE EXTENDED surfaces both). + for (key, value) in status.properties.iter_mut() { + if key.eq_ignore_ascii_case("location") + && let Some(abfss) = normalize_onelake_location(value) + { + *value = abfss; + } + } + status } - fn catalog_name(&self) -> String { - format!("{}.{}", self.config.item_name, self.config.item_type) - } - - async fn get_client(&self) -> CatalogResult<&Arc> { - self.client - .get_or_try_init(|| async { - // Priority: 1) config token, 2) env var, 3) Azure CLI - let token = if let Some(token) = &self.config.bearer_token { - token.clone() - } else if let Ok(token) = std::env::var("AZURE_STORAGE_TOKEN") { - token - } else if let Ok(token) = std::env::var("AZURE_ACCESS_TOKEN") { - token - } else { - // Fall back to Azure CLI - get_azure_cli_token().await? - }; - - let mut headers = reqwest::header::HeaderMap::new(); - let header_value = reqwest::header::HeaderValue::from_str(&format!( - "Bearer {token}" - )) - .map_err(|e| CatalogError::External(format!("Invalid bearer token: {e}")))?; - headers.insert(reqwest::header::AUTHORIZATION, header_value); - - let client = Client::builder() - .default_headers(headers) - .build() - .map_err(|e| { - CatalogError::External(format!("Failed to build HTTP client: {e}")) - })?; - - Ok(Arc::new(client)) - }) - .await + fn normalize_table_status(&self, mut status: TableStatus) -> TableStatus { + if matches!(self.config.api, OneLakeApiKind::Delta) { + status.database = self.normalize_delta_database(status.database); + if let TableKind::Table { location, .. } = &mut status.kind + && let Some(url) = location.take() + { + *location = Some(normalize_onelake_location(&url).unwrap_or(url)); + } + } + status } - fn schema_info_to_database_status(&self, info: SchemaInfo) -> DatabaseStatus { - let schema_name = info.name.unwrap_or_else(|| "dbo".to_string()); - DatabaseStatus { - catalog: self.name.clone(), - database: vec![schema_name], - comment: info.comment, - location: None, - properties: vec![], + fn normalize_delta_database(&self, database: Vec) -> Vec { + if database.first().is_some_and(|x| x == &self.catalog_name()) { + database.into_iter().skip(1).collect() + } else { + database } } - fn table_info_to_table_status(&self, info: TableInfo) -> CatalogResult { - let name = info.name.unwrap_or_default(); - let schema_name = info.schema_name.unwrap_or_else(|| "dbo".to_string()); - let format = info - .data_source_format - .map(|f| f.to_lowercase()) - .unwrap_or_else(|| "delta".to_string()); - - // Convert https:// URL to abfss:// for Delta Lake access - let location = info - .storage_location - .and_then(|url| convert_to_abfss_url(&url).or(Some(url))); + fn normalize_database_statuses(&self, statuses: Vec) -> Vec { + statuses + .into_iter() + .map(|status| self.normalize_database_status(status)) + .collect() + } - let columns = info - .columns - .unwrap_or_default() + fn normalize_table_statuses(&self, statuses: Vec) -> Vec { + statuses .into_iter() - .map(|col| { - // Prefer type_name over type_text (OneLake API often returns type_text as null) - let type_str = col - .type_name - .or(col.type_text) - .unwrap_or_else(|| "string".to_string()); - let data_type = parse_type_text(&type_str); - TableColumnStatus { - name: col.name.unwrap_or_default(), - data_type, - nullable: col.nullable.unwrap_or(true), - comment: col.comment, - default: None, - generated_always_as: None, - identity: None, - is_partition: false, - is_bucket: false, - is_cluster: false, - } - }) - .collect(); - - Ok(TableStatus { - catalog: Some(self.name.clone()), - database: vec![schema_name], - name, - kind: TableKind::Table { - columns, - comment: info.comment, - constraints: vec![], - location, - format, - partition_by: vec![], - sort_by: vec![], - bucket_by: None, - properties: vec![], - is_external: true, - }, - }) + .map(|status| self.normalize_table_status(status)) + .collect() } } -/// Simple type text parser - converts Unity Catalog type strings to Arrow DataType -// TODO: Add support for complex types (array, map, struct) if OneLake returns them in JSON format. -fn parse_type_text(type_text: &str) -> arrow::datatypes::DataType { - use arrow::datatypes::DataType; +// OneLake reports locations in different forms depending on the API: +// - Delta (Unity) returns an absolute `https://` URL: +// From: https://onelake.dfs.fabric.microsoft.com/workspace/item.ItemType/Tables/schema/table +// To: abfss://workspace@onelake.dfs.fabric.microsoft.com/item.ItemType/Tables/schema/table +// - Iceberg (REST) returns the namespace `location` as a workspace-relative path: +// From: workspace/item.ItemType/Tables/schema +// To: abfss://workspace@onelake.dfs.fabric.microsoft.com/item.ItemType/Tables/schema +fn normalize_onelake_location(location: &str) -> Option { + let trimmed = location.trim(); + let relative = + if let Some(rest) = trimmed.strip_prefix("https://onelake.dfs.fabric.microsoft.com/") { + // OneLake `https://` form. + rest + } else if Url::parse(trimmed) + .ok() + .is_some_and(|url| url.scheme().len() > 1) + { + // Already absolute with another scheme (e.g., `abfss://`, `file://`, ...). + return None; + } else { + // Workspace-relative form (Iceberg namespace location). + trimmed + }; + let (workspace, rest) = relative.split_once('/')?; + (!workspace.is_empty() && !rest.is_empty()) + .then(|| format!("abfss://{workspace}@onelake.dfs.fabric.microsoft.com/{rest}")) +} - let lower = type_text.to_lowercase(); +// `/.` for friendly names, or `/` for GUIDs. +fn onelake_item_path(workspace: &str, item_name: &str, item_type: Option<&str>) -> String { + match item_type { + Some(item_type) => format!("{workspace}/{item_name}.{item_type}"), + None => format!("{workspace}/{item_name}"), + } +} - // Handle decimal(precision, scale) format - if lower.starts_with("decimal") { - if let Some(params) = lower - .strip_prefix("decimal(") - .and_then(|s| s.strip_suffix(')')) - { - let parts: Vec<&str> = params.split(',').collect(); - if parts.len() == 2 { - if let (Ok(precision), Ok(scale)) = - (parts[0].trim().parse::(), parts[1].trim().parse::()) - { - return DataType::Decimal128(precision, scale); - } - } - } - // Default decimal if parsing fails - return DataType::Decimal128(38, 18); - } - - match lower.as_str() { - "boolean" | "bool" => DataType::Boolean, - "byte" | "tinyint" => DataType::Int8, - "short" | "smallint" => DataType::Int16, - "int" | "integer" => DataType::Int32, - "long" | "bigint" => DataType::Int64, - "float" | "real" => DataType::Float32, - "double" => DataType::Float64, - "string" | "varchar" | "text" => DataType::Utf8, - "binary" => DataType::Binary, - "date" => DataType::Date32, - "timestamp" | "timestamp_ntz" => { - DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, None) - } - _ => DataType::Utf8, // Default to string for unknown types +// `.` for friendly names, or `` for GUIDs. +fn onelake_catalog_name(item_name: &str, item_type: Option<&str>) -> String { + match item_type { + Some(item_type) => format!("{item_name}.{item_type}"), + None => item_name.to_string(), } } @@ -331,238 +233,211 @@ impl CatalogProvider for OneLakeCatalogProvider { async fn create_database( &self, - _database: &Namespace, - _options: CreateDatabaseOptions, + database: &Namespace, + options: CreateDatabaseOptions, ) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support creating databases".to_string(), - )) + let status = self.inner.create_database(database, options).await?; + Ok(self.normalize_database_status(status)) } async fn get_database(&self, database: &Namespace) -> CatalogResult { - let schema_name = Self::schema_name(database)?; - let client = self.get_client().await?; - - // OneLake API requires full qualified schema name: catalog.schema - let full_schema_name = format!("{}.{}", self.catalog_name(), schema_name); - let url = format!( - "{}/api/2.1/unity-catalog/schemas/{}", - self.base_url(), - full_schema_name - ); - - let response = client - .head(&url) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to check schema: {e}")))?; - - if response.status().is_success() { - Ok(DatabaseStatus { - catalog: self.name.clone(), - database: vec![schema_name], - comment: None, - location: None, - properties: vec![], - }) - } else if response.status().as_u16() == 404 { - Err(CatalogError::NotFound(CatalogObject::Schema, schema_name)) - } else { - Err(CatalogError::External(format!( - "Failed to get schema: HTTP {}", - response.status() - ))) - } + let status = self.inner.get_database(database).await?; + Ok(self.normalize_database_status(status)) } async fn list_databases( &self, - _prefix: Option<&Namespace>, + prefix: Option<&Namespace>, ) -> CatalogResult> { - let client = self.get_client().await?; - - // TODO: Use Url methods to construct URLs in case catalog/schema/table names contain special characters. - let url = format!( - "{}/api/2.1/unity-catalog/schemas?catalog_name={}", - self.base_url(), - self.catalog_name() - ); - - let response = client - .get(&url) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to list schemas: {e}")))?; - - if !response.status().is_success() { - return Err(CatalogError::External(format!( - "Failed to list schemas: HTTP {}", - response.status() - ))); - } - - let list_response: ListSchemasResponse = response.json().await.map_err(|e| { - CatalogError::External(format!("Failed to parse schemas response: {e}")) - })?; - - Ok(list_response - .schemas - .into_iter() - .map(|s| self.schema_info_to_database_status(s)) - .collect()) + let statuses = self.inner.list_databases(prefix).await?; + Ok(self.normalize_database_statuses(statuses)) } async fn drop_database( &self, - _database: &Namespace, - _options: DropDatabaseOptions, + database: &Namespace, + options: DropDatabaseOptions, ) -> CatalogResult<()> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support dropping databases".to_string(), - )) + self.inner.drop_database(database, options).await } async fn create_table( &self, - _database: &Namespace, - _table: &str, - _options: CreateTableOptions, + database: &Namespace, + table: &str, + options: CreateTableOptions, ) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support creating tables via API".to_string(), - )) + let status = self.inner.create_table(database, table, options).await?; + Ok(self.normalize_table_status(status)) } - async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { - let schema_name = Self::schema_name(database)?; - let client = self.get_client().await?; - - let full_table_name = format!("{}.{}.{}", self.catalog_name(), schema_name, table); - let url = format!( - "{}/api/2.1/unity-catalog/tables/{}", - self.base_url(), - full_table_name - ); - - let response = client - .get(&url) - .send() - .await - .map_err(|e| CatalogError::External(format!("Failed to get table: {e}")))?; - - if response.status().as_u16() == 404 { - return Err(CatalogError::NotFound( - CatalogObject::Table, - table.to_string(), - )); - } + fn create_table_metadata_requirement( + &self, + options: &CreateTableOptions, + ) -> CatalogResult { + self.inner.create_table_metadata_requirement(options) + } - if !response.status().is_success() { - return Err(CatalogError::External(format!( - "Failed to get table: HTTP {}", - response.status() - ))); - } + fn lakehouse_capabilities(&self) -> Vec { + self.inner.lakehouse_capabilities() + } - let table_info: TableInfo = response - .json() + async fn resolve_lakehouse_table( + &self, + database: &Namespace, + table: &str, + request: ResolveLakehouseTableRequest, + ) -> CatalogResult { + self.inner + .resolve_lakehouse_table(database, table, request) .await - .map_err(|e| CatalogError::External(format!("Failed to parse table response: {e}")))?; + } - self.table_info_to_table_status(table_info) + async fn plan_lakehouse_create( + &self, + database: &Namespace, + table: &str, + request: LakehouseCreateRequest, + ) -> CatalogResult { + self.inner + .plan_lakehouse_create(database, table, request) + .await } - async fn list_tables(&self, database: &Namespace) -> CatalogResult> { - let schema_name = Self::schema_name(database)?; - let client = self.get_client().await?; - - let url = format!( - "{}/api/2.1/unity-catalog/tables?catalog_name={}&schema_name={}", - self.base_url(), - self.catalog_name(), - schema_name - ); + async fn begin_table_access( + &self, + database: &Namespace, + table: &str, + request: BeginTableAccessRequest, + ) -> CatalogResult { + self.inner + .begin_table_access(database, table, request) + .await + } - let response = client - .get(&url) - .send() + async fn plan_lakehouse_scan( + &self, + database: &Namespace, + table: &str, + request: LakehouseScanPlanningRequest, + ) -> CatalogResult { + self.inner + .plan_lakehouse_scan(database, table, request) .await - .map_err(|e| CatalogError::External(format!("Failed to list tables: {e}")))?; + } - if !response.status().is_success() { - return Err(CatalogError::External(format!( - "Failed to list tables: HTTP {}", - response.status() - ))); - } + async fn commit_lakehouse_table( + &self, + database: &Namespace, + table: &str, + request: LakehouseCommitRequest, + ) -> CatalogResult { + self.inner + .commit_lakehouse_table(database, table, request) + .await + } - let list_response: ListTablesResponse = response - .json() + async fn get_delta_ratified_commits( + &self, + database: &Namespace, + table: &str, + request: DeltaRatifiedCommitRequest, + ) -> CatalogResult { + self.inner + .get_delta_ratified_commits(database, table, request) .await - .map_err(|e| CatalogError::External(format!("Failed to parse tables response: {e}")))?; + } - list_response - .tables - .into_iter() - .map(|t| self.table_info_to_table_status(t)) - .collect() + async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { + let status = self.inner.get_table(database, table).await?; + Ok(self.normalize_table_status(status)) + } + + async fn list_tables(&self, database: &Namespace) -> CatalogResult> { + let statuses = self.inner.list_tables(database).await?; + Ok(self.normalize_table_statuses(statuses)) } async fn drop_table( &self, - _database: &Namespace, - _table: &str, - _options: DropTableOptions, + database: &Namespace, + table: &str, + options: DropTableOptions, ) -> CatalogResult<()> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support dropping tables via API".to_string(), - )) + self.inner.drop_table(database, table, options).await } async fn alter_table( &self, - _database: &Namespace, - _table: &str, - _options: AlterTableOptions, + database: &Namespace, + table: &str, + options: AlterTableOptions, ) -> CatalogResult<()> { - // OneLake tables commonly use Delta storage, and property updates may already - // be committed at the storage layer before the catalog provider is called. - // Until OneLake REST propagation is implemented, treat this as a no-op so we - // do not report a failure after the underlying table has already been altered. - Ok(()) + self.inner.alter_table(database, table, options).await } async fn create_view( &self, - _database: &Namespace, - _view: &str, - _options: CreateViewOptions, + database: &Namespace, + view: &str, + options: CreateViewOptions, ) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + let status = self.inner.create_view(database, view, options).await?; + Ok(self.normalize_table_status(status)) } - async fn get_view(&self, _database: &Namespace, _view: &str) -> CatalogResult { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + async fn get_view(&self, database: &Namespace, view: &str) -> CatalogResult { + let status = self.inner.get_view(database, view).await?; + Ok(self.normalize_table_status(status)) } - async fn list_views(&self, _database: &Namespace) -> CatalogResult> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + async fn list_views(&self, database: &Namespace) -> CatalogResult> { + let statuses = self.inner.list_views(database).await?; + Ok(self.normalize_table_statuses(statuses)) } async fn drop_view( &self, - _database: &Namespace, - _view: &str, - _options: DropViewOptions, + database: &Namespace, + view: &str, + options: DropViewOptions, ) -> CatalogResult<()> { - Err(CatalogError::NotSupported( - "OneLake catalog does not support views".to_string(), - )) + self.inner.drop_view(database, view, options).await + } +} + +#[cfg(test)] +mod tests { + use super::normalize_onelake_location; + + #[test] + fn normalize_onelake_location_handles_each_form() { + // Iceberg: workspace-relative namespace location -> abfss (the reported bug). + assert_eq!( + normalize_onelake_location("OneLake_LakeSail_Testing/LakeSail.Lakehouse/Tables/dbo") + .as_deref(), + Some( + "abfss://OneLake_LakeSail_Testing@onelake.dfs.fabric.microsoft.com/LakeSail.Lakehouse/Tables/dbo" + ), + ); + // Delta/Unity: absolute https OneLake URL -> abfss (unchanged from before). + assert_eq!( + normalize_onelake_location( + "https://onelake.dfs.fabric.microsoft.com/ws/item.Lakehouse/Tables/dbo/t", + ) + .as_deref(), + Some("abfss://ws@onelake.dfs.fabric.microsoft.com/item.Lakehouse/Tables/dbo/t"), + ); + // Already absolute -> None so the caller keeps the original (never double-convert). + assert_eq!( + normalize_onelake_location("abfss://ws@onelake.dfs.fabric.microsoft.com/x"), + None, + ); + assert_eq!(normalize_onelake_location("s3://bucket/x"), None); + // Single-slash scheme: `Url::parse` should detect it. + assert_eq!(normalize_onelake_location("file:/tmp/onelake"), None); + // Degenerate inputs -> no conversion. + assert_eq!(normalize_onelake_location("single-segment"), None); + assert_eq!(normalize_onelake_location("ws/"), None); } } diff --git a/crates/sail-catalog-system/src/lib.rs b/crates/sail-catalog-system/src/lib.rs index 905b13f07..fbc7cdfa3 100644 --- a/crates/sail-catalog-system/src/lib.rs +++ b/crates/sail-catalog-system/src/lib.rs @@ -5,4 +5,4 @@ pub mod provider; pub mod service; pub mod table_source; -pub use provider::{SystemCatalogProvider, SYSTEM_CATALOG_NAME}; +pub use provider::{SYSTEM_CATALOG_NAME, SystemCatalogProvider}; diff --git a/crates/sail-catalog-system/src/physical_plan.rs b/crates/sail-catalog-system/src/physical_plan.rs index f8c9f8ab6..54567683b 100644 --- a/crates/sail-catalog-system/src/physical_plan.rs +++ b/crates/sail-catalog-system/src/physical_plan.rs @@ -1,7 +1,7 @@ use std::fmt::Formatter; use std::sync::Arc; -use datafusion::common::{exec_err, plan_err, Result}; +use datafusion::common::{Result, exec_err, plan_err}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; diff --git a/crates/sail-catalog-system/src/predicate.rs b/crates/sail-catalog-system/src/predicate.rs index d0a4e41da..6cd0aeb77 100644 --- a/crates/sail-catalog-system/src/predicate.rs +++ b/crates/sail-catalog-system/src/predicate.rs @@ -6,10 +6,10 @@ use datafusion::arrow::array::{Array, BooleanArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{internal_datafusion_err, Result, ScalarValue}; +use datafusion::common::{Result, ScalarValue, internal_datafusion_err}; use datafusion::logical_expr::{Expr, Operator}; -use datafusion::physical_expr::expressions::BinaryExpr; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::expressions::BinaryExpr; use datafusion::physical_plan::internal_err; use sail_common_datafusion::system::predicate::Predicate; @@ -66,11 +66,11 @@ pub fn is_column_logical_predicate(expr: &Expr, column: &str) -> Result { let mut valid = true; expr.apply(|e| { - if let Expr::Column(Column { name, .. }) = e { - if name != column { - valid = false; - return Ok(TreeNodeRecursion::Stop); - } + if let Expr::Column(Column { name, .. }) = e + && name != column + { + valid = false; + return Ok(TreeNodeRecursion::Stop); } Ok(TreeNodeRecursion::Continue) })?; @@ -82,11 +82,11 @@ pub fn is_column_physical_predicate(expr: &Arc, column: &str) let mut valid = true; expr.apply(|e| { - if let Some(col) = e.downcast_ref::() { - if col.name() != column { - valid = false; - return Ok(TreeNodeRecursion::Stop); - } + if let Some(col) = e.downcast_ref::() + && col.name() != column + { + valid = false; + return Ok(TreeNodeRecursion::Stop); } Ok(TreeNodeRecursion::Continue) })?; @@ -197,8 +197,8 @@ mod tests { use std::sync::Arc; use datafusion::logical_expr::Operator; - use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal}; use datafusion::physical_expr::PhysicalExpr; + use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal}; use datafusion::scalar::ScalarValue; use super::ArrowPredicateEvaluator; diff --git a/crates/sail-catalog-system/src/provider.rs b/crates/sail-catalog-system/src/provider.rs index 0ba7fb701..622bcf303 100644 --- a/crates/sail-catalog-system/src/provider.rs +++ b/crates/sail-catalog-system/src/provider.rs @@ -98,10 +98,10 @@ impl CatalogProvider for SystemCatalogProvider { async fn get_database(&self, database: &Namespace) -> CatalogResult { let Namespace { head, tail } = database; - if tail.is_empty() { - if let Some(db) = SystemDatabase::get(head) { - return Self::get_database_status(database, &db); - } + if tail.is_empty() + && let Some(db) = SystemDatabase::get(head) + { + return Self::get_database_status(database, &db); } Err(CatalogError::NotFound( CatalogObject::Database, @@ -150,18 +150,18 @@ impl CatalogProvider for SystemCatalogProvider { async fn get_table(&self, database: &Namespace, table: &str) -> CatalogResult { let Namespace { head, tail } = database; - if tail.is_empty() { - if let Some(db) = SystemDatabase::get(head) { - for t in db.tables() { - if table.eq_ignore_ascii_case(t.name()) { - return Self::get_table_status(database, table, *t); - } + if tail.is_empty() + && let Some(db) = SystemDatabase::get(head) + { + for t in db.tables() { + if table.eq_ignore_ascii_case(t.name()) { + return Self::get_table_status(database, table, *t); } - return Err(CatalogError::NotFound( - CatalogObject::Table, - table.to_string(), - )); } + return Err(CatalogError::NotFound( + CatalogObject::Table, + table.to_string(), + )); } Err(CatalogError::NotFound( CatalogObject::Database, @@ -171,15 +171,15 @@ impl CatalogProvider for SystemCatalogProvider { async fn list_tables(&self, database: &Namespace) -> CatalogResult> { let Namespace { head, tail } = database; - if tail.is_empty() { - if let Some(db) = SystemDatabase::get(head) { - let mut result = vec![]; - for t in db.tables() { - let status = Self::get_table_status(database, t.name(), *t)?; - result.push(status); - } - return Ok(result); + if tail.is_empty() + && let Some(db) = SystemDatabase::get(head) + { + let mut result = vec![]; + for t in db.tables() { + let status = Self::get_table_status(database, t.name(), *t)?; + result.push(status); } + return Ok(result); } Err(CatalogError::NotFound( CatalogObject::Database, diff --git a/crates/sail-catalog-system/src/service.rs b/crates/sail-catalog-system/src/service.rs index d847371ac..a00bc31d0 100644 --- a/crates/sail-catalog-system/src/service.rs +++ b/crates/sail-catalog-system/src/service.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::{internal_datafusion_err, Result}; +use datafusion::common::{Result, internal_datafusion_err}; use datafusion::physical_expr::PhysicalExpr; use sail_common_datafusion::array::serde::ArrowSerializer; use sail_common_datafusion::extension::SessionExtension; diff --git a/crates/sail-catalog-unity/Cargo.toml b/crates/sail-catalog-unity/Cargo.toml index 7f1682264..db391383d 100644 --- a/crates/sail-catalog-unity/Cargo.toml +++ b/crates/sail-catalog-unity/Cargo.toml @@ -28,8 +28,9 @@ arrow = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } -progenitor-client = { workspace = true } +percent-encoding = { workspace = true } secrecy = { workspace = true } chrono = { workspace = true } @@ -40,7 +41,4 @@ wiremock = { workspace = true } tempfile = { workspace = true } [build-dependencies] -progenitor = { workspace = true } -prettyplease = { workspace = true } -serde_yaml = { workspace = true } -syn = { workspace = true } +sail-build-scripts = { path = "../sail-build-scripts" } diff --git a/crates/sail-catalog-unity/build.rs b/crates/sail-catalog-unity/build.rs index d1735fa92..c3ef4ce01 100644 --- a/crates/sail-catalog-unity/build.rs +++ b/crates/sail-catalog-unity/build.rs @@ -9,43 +9,20 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -fn build_unity_catalog() -> Result<(), Box> { - let src = "spec/unity-catalog-all.yaml"; - println!("cargo:rerun-if-changed={src}"); - let file = std::fs::File::open(src)?; - let spec = serde_yaml::from_reader(file)?; - - let mut settings = progenitor::GenerationSettings::new(); - settings.with_interface(progenitor::InterfaceStyle::Builder); - let mut generator = progenitor::Generator::new(&settings); - let tokens = generator.generate_tokens(&spec)?; - let ast = syn::parse2(tokens)?; - let content = prettyplease::unparse(&ast); - - let mut out_file = std::path::Path::new(&std::env::var("OUT_DIR")?).to_path_buf(); - out_file.push("unity_catalog.rs"); - - let content = content - .replace( - "pub fn new(baseurl: &str) -> Self {", - "pub fn new(baseurl: &str) -> Result {", - ) - .replace( - "Self::new_with_client(baseurl, client.build().unwrap())", - "Ok(Self::new_with_client(baseurl, client.build()?))", - ) - // We need to prevent the code examples from being treated as ignored tests. - // These code snippets do not compile when running `cargo test -- --ignored`. - .replace("```ignore", "```notrust"); - - std::fs::write(out_file, content)?; - - Ok(()) -} +use sail_build_scripts::openapi::generator::{OpenApiConfig, generate_openapi_client}; fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=build.rs"); - build_unity_catalog()?; + + generate_openapi_client( + "spec/unity-catalog-all.yaml", + std::path::Path::new(&std::env::var("OUT_DIR")?).join("unity_catalog_gen.rs"), + OpenApiConfig { + excluded_operations: [].into_iter().collect(), + excluded_schemas: ["ColumnTypeName".to_owned()].into_iter().collect(), + }, + )?; + Ok(()) } diff --git a/crates/sail-catalog-unity/src/config.rs b/crates/sail-catalog-unity/src/config.rs index 57cd50f3a..3a6c447ba 100644 --- a/crates/sail-catalog-unity/src/config.rs +++ b/crates/sail-catalog-unity/src/config.rs @@ -22,7 +22,6 @@ use std::collections::HashMap; use std::str::FromStr; use sail_catalog::error::{CatalogError, CatalogResult}; -use sail_catalog::utils::quote_name_if_needed; use secrecy::{ExposeSecret, SecretString}; use crate::credential::{ @@ -221,7 +220,6 @@ impl AsRef for UnityCatalogConfigKey { #[derive(Debug)] pub struct UnityCatalogConfig { - pub default_catalog: String, pub uri: String, pub bearer_token: Option, pub client_id: Option, @@ -234,7 +232,6 @@ pub struct UnityCatalogConfig { pub federated_token_file: Option, pub use_azure_cli: bool, pub allow_http_url: bool, - pub credential_provider: Option, } fn str_is_truthy(value: &str) -> bool { @@ -248,15 +245,11 @@ fn str_is_truthy(value: &str) -> bool { impl UnityCatalogConfig { pub fn new( - default_catalog: Option, uri: Option, token: &Option, options: Option>, ) -> CatalogResult { let mut config = Self { - default_catalog: quote_name_if_needed( - &default_catalog.unwrap_or_else(|| "unity".to_string()), - ), uri: uri.unwrap_or_else(|| DEFAULT_URI.to_string()), bearer_token: None, client_id: None, @@ -269,7 +262,6 @@ impl UnityCatalogConfig { federated_token_file: None, use_azure_cli: false, allow_http_url: false, - credential_provider: None, }; if let Some(token) = token { @@ -283,8 +275,6 @@ impl UnityCatalogConfig { config.apply_env()?; - config.credential_provider = config.get_credential_provider(); - Ok(config) } @@ -292,15 +282,13 @@ impl UnityCatalogConfig { for (os_key, os_value) in std::env::vars_os() { if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { let key = key.trim(); - if key.to_ascii_uppercase().starts_with("UNITY_") + if (key.to_ascii_uppercase().starts_with("UNITY_") || key.to_ascii_uppercase().starts_with("DATABRICKS_") - || key.to_ascii_uppercase().starts_with("UC_") - { - if let Ok(config_key) = + || key.to_ascii_uppercase().starts_with("UC_")) + && let Ok(config_key) = UnityCatalogConfigKey::from_str(&key.to_ascii_lowercase()) - { - self.set_option(config_key, value.to_string())?; - } + { + self.set_option(config_key, value.to_string())?; } } } @@ -364,19 +352,21 @@ impl UnityCatalogConfig { )); } - if let (Some(client_id), Some(client_secret)) = (&self.client_id, &self.client_secret) { - if let Some(authority_id) = &self.authority_id { - return Some(CredentialProvider::TokenCredential( - TokenCache::default(), - Box::new(ClientSecretOAuthProvider::new( - client_id, - client_secret.expose_secret(), - authority_id, - self.authority_host.as_ref(), - )), - )); - } + if let (Some(client_id), Some(client_secret), Some(authority_id)) = + (&self.client_id, &self.client_secret, &self.authority_id) + { + return Some(CredentialProvider::TokenCredential( + TokenCache::default(), + Box::new(ClientSecretOAuthProvider::new( + client_id, + client_secret.expose_secret(), + authority_id, + self.authority_host.as_ref(), + )), + )); + } + if let (Some(client_id), Some(client_secret)) = (&self.client_id, &self.client_secret) { return Some(CredentialProvider::TokenCredential( TokenCache::default(), Box::new(WorkspaceOAuthProvider::new( diff --git a/crates/sail-catalog-unity/src/credential.rs b/crates/sail-catalog-unity/src/credential.rs index a35ee9c2d..4bc0907ee 100644 --- a/crates/sail-catalog-unity/src/credential.rs +++ b/crates/sail-catalog-unity/src/credential.rs @@ -21,11 +21,12 @@ use std::process::Command; use std::time::{Duration, Instant}; -use reqwest::header::{HeaderValue, ACCEPT}; +use reqwest::header::{ACCEPT, HeaderValue}; use reqwest::{Client, Method, Response}; +use sail_catalog::credentials::CatalogCredentials; use sail_catalog::error::{CatalogError, CatalogResult}; -use serde::de::DeserializeOwned; use serde::Deserialize; +use serde::de::DeserializeOwned; use crate::token::{TemporaryToken, TokenCache}; @@ -69,6 +70,24 @@ pub enum CredentialProvider { TokenCredential(TokenCache, Box), } +#[async_trait::async_trait] +impl CatalogCredentials for CredentialProvider { + async fn retrieve(&self) -> CatalogResult> { + match self { + CredentialProvider::BearerToken(token) => Ok(Some(token.clone())), + CredentialProvider::TokenCredential(cache, cred) => { + let oauth_client = Client::builder().build().map_err(|e| { + CatalogError::External(format!("Failed to build OAuth client: {e}")) + })?; + let token = cache + .get_or_insert_with(|| cred.fetch_token(&oauth_client)) + .await?; + Ok(Some(token)) + } + } + } +} + #[derive(Deserialize, Debug)] struct TokenResponse { access_token: String, @@ -424,12 +443,12 @@ impl ImdsManagedIdentityOAuthProvider { msi_res_id, } } -} -#[async_trait::async_trait] -impl TokenCredential for ImdsManagedIdentityOAuthProvider { - /// Fetch a token - async fn fetch_token(&self, client: &Client) -> Result, CatalogError> { + async fn fetch_token_with_identity_header( + &self, + client: &Client, + identity_header: Option<&str>, + ) -> Result, CatalogError> { let resource_scope = format!("{DATABRICKS_RESOURCE_SCOPE}/.default"); let mut query_items = vec![ ("api-version", MSI_API_VERSION), @@ -455,8 +474,8 @@ impl TokenCredential for ImdsManagedIdentityOAuthProvider { .header("metadata", "true") .query(&query_items); - if let Ok(val) = std::env::var(MSI_SECRET_ENV_KEY) { - builder = builder.header("x-identity-header", val); + if let Some(value) = identity_header { + builder = builder.header("x-identity-header", value); }; let response = builder @@ -473,6 +492,16 @@ impl TokenCredential for ImdsManagedIdentityOAuthProvider { } } +#[async_trait::async_trait] +impl TokenCredential for ImdsManagedIdentityOAuthProvider { + /// Fetch a token + async fn fetch_token(&self, client: &Client) -> Result, CatalogError> { + let identity_header = std::env::var(MSI_SECRET_ENV_KEY).ok(); + self.fetch_token_with_identity_header(client, identity_header.as_deref()) + .await + } +} + #[expect(clippy::unwrap_used)] #[cfg(test)] mod tests { @@ -486,9 +515,6 @@ mod tests { #[tokio::test] async fn test_managed_identity() { let server = MockServer::start().await; - - std::env::set_var(MSI_SECRET_ENV_KEY, "env-secret"); - let client = Client::new(); Mock::given(method("GET")) @@ -515,7 +541,10 @@ mod tests { Some(format!("{}/metadata/identity/oauth2/token", server.uri())), ); - let token = credential.fetch_token(&client).await.unwrap(); + let token = credential + .fetch_token_with_identity_header(&client, Some("env-secret")) + .await + .unwrap(); assert_eq!(&token.token, "TOKEN"); } diff --git a/crates/sail-catalog-unity/src/data_type.rs b/crates/sail-catalog-unity/src/data_type.rs index a669c5fbf..b5e309708 100644 --- a/crates/sail-catalog-unity/src/data_type.rs +++ b/crates/sail-catalog-unity/src/data_type.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; use std::sync::Arc; use arrow::datatypes::{ - DataType, Field, Fields, IntervalUnit, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, + DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DataType, Field, Fields, IntervalUnit, TimeUnit, }; use sail_catalog::error::{CatalogError, CatalogResult}; use sail_common::spec::{ @@ -22,7 +22,7 @@ use sail_common::spec::{ SAIL_SPARK_UDT_METADATA_KEY, SPARK_METADATA_JSON_KEY, }; -use crate::unity::types; +use crate::r#gen; // There is no clarity on the expected format for `type_json` and `type_text`. // Open source code and docs have various inconsistencies and contradictions. @@ -31,7 +31,7 @@ use crate::unity::types; pub(crate) struct UnityColumnType { pub type_text: String, pub type_json: serde_json::Value, - pub type_name: types::ColumnTypeName, + pub type_name: r#gen::ColumnTypeName, } pub(crate) fn unity_struct_field_type_json( @@ -199,102 +199,102 @@ pub(crate) fn data_type_to_unity_type(data_type: &DataType) -> CatalogResult Ok(UnityColumnType { - type_text: types::ColumnTypeName::Null.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Null.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Null.to_string().to_lowercase(), + r#gen::ColumnTypeName::Null.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Null, + type_name: r#gen::ColumnTypeName::Null, }), DataType::Boolean => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Boolean.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Boolean.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Boolean.to_string().to_lowercase(), + r#gen::ColumnTypeName::Boolean.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Boolean, + type_name: r#gen::ColumnTypeName::Boolean, }), DataType::Int8 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Byte.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Byte.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Byte.to_string().to_lowercase(), + r#gen::ColumnTypeName::Byte.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Byte, + type_name: r#gen::ColumnTypeName::Byte, }), DataType::Int16 | DataType::UInt8 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Short.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Short.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Short.to_string().to_lowercase(), + r#gen::ColumnTypeName::Short.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Short, + type_name: r#gen::ColumnTypeName::Short, }), DataType::Int32 | DataType::UInt16 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Int.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Int.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Int.to_string().to_lowercase(), + r#gen::ColumnTypeName::Int.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Int, + type_name: r#gen::ColumnTypeName::Int, }), DataType::Int64 | DataType::UInt32 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Long.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Long.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Long.to_string().to_lowercase(), + r#gen::ColumnTypeName::Long.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Long, + type_name: r#gen::ColumnTypeName::Long, }), DataType::Float32 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Float.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Float.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Float.to_string().to_lowercase(), + r#gen::ColumnTypeName::Float.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Float, + type_name: r#gen::ColumnTypeName::Float, }), DataType::Float64 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Double.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Double.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Double.to_string().to_lowercase(), + r#gen::ColumnTypeName::Double.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Double, + type_name: r#gen::ColumnTypeName::Double, }), DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Timestamp.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Timestamp.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Timestamp.to_string().to_lowercase(), + r#gen::ColumnTypeName::Timestamp.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Timestamp, + type_name: r#gen::ColumnTypeName::Timestamp, }), DataType::Timestamp(TimeUnit::Microsecond, None) => Ok(UnityColumnType { - type_text: types::ColumnTypeName::TimestampNtz + type_text: r#gen::ColumnTypeName::TimestampNtz .to_string() .to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::TimestampNtz + r#gen::ColumnTypeName::TimestampNtz .to_string() .to_lowercase(), ), - type_name: types::ColumnTypeName::TimestampNtz, + type_name: r#gen::ColumnTypeName::TimestampNtz, }), DataType::Date32 => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Date.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Date.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Date.to_string().to_lowercase(), + r#gen::ColumnTypeName::Date.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Date, + type_name: r#gen::ColumnTypeName::Date, }), DataType::Binary | DataType::FixedSizeBinary(_) | DataType::LargeBinary | DataType::BinaryView => Ok(UnityColumnType { - type_text: types::ColumnTypeName::Binary.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::Binary.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::Binary.to_string().to_lowercase(), + r#gen::ColumnTypeName::Binary.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::Binary, + type_name: r#gen::ColumnTypeName::Binary, }), DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(UnityColumnType { - type_text: types::ColumnTypeName::String.to_string().to_lowercase(), + type_text: r#gen::ColumnTypeName::String.to_string().to_lowercase(), type_json: serde_json::Value::String( - types::ColumnTypeName::String.to_string().to_lowercase(), + r#gen::ColumnTypeName::String.to_string().to_lowercase(), ), - type_name: types::ColumnTypeName::String, + type_name: r#gen::ColumnTypeName::String, }), DataType::Duration(TimeUnit::Microsecond) => { let type_json = serde_json::json!({ @@ -306,9 +306,9 @@ pub(crate) fn data_type_to_unity_type(data_type: &DataType) -> CatalogResult { @@ -328,9 +328,9 @@ pub(crate) fn data_type_to_unity_type(data_type: &DataType) -> CatalogResult CatalogResult { @@ -363,7 +363,7 @@ pub(crate) fn data_type_to_unity_type(data_type: &DataType) -> CatalogResult CatalogResult { @@ -417,7 +417,7 @@ pub(crate) fn data_type_to_unity_type(data_type: &DataType) -> CatalogResult CatalogResult CatalogResult, + type_name: Option, type_json: Option, type_text: Option, type_precision: Option, @@ -492,25 +492,25 @@ pub(crate) fn unity_type_to_data_type( // 3. Parse interval type for `Interval` if type_interval_type and type_scale aren't provided. if let Some(type_name) = type_name { match type_name { - types::ColumnTypeName::Boolean => Ok(DataType::Boolean), - types::ColumnTypeName::Byte => Ok(DataType::Int8), - types::ColumnTypeName::Short => Ok(DataType::Int16), - types::ColumnTypeName::Int => Ok(DataType::Int32), - types::ColumnTypeName::Long => Ok(DataType::Int64), - types::ColumnTypeName::Float => Ok(DataType::Float32), - types::ColumnTypeName::Double => Ok(DataType::Float64), - types::ColumnTypeName::Date => Ok(DataType::Date32), - types::ColumnTypeName::Timestamp => Ok(DataType::Timestamp( + r#gen::ColumnTypeName::Boolean => Ok(DataType::Boolean), + r#gen::ColumnTypeName::Byte => Ok(DataType::Int8), + r#gen::ColumnTypeName::Short => Ok(DataType::Int16), + r#gen::ColumnTypeName::Int => Ok(DataType::Int32), + r#gen::ColumnTypeName::Long => Ok(DataType::Int64), + r#gen::ColumnTypeName::Float => Ok(DataType::Float32), + r#gen::ColumnTypeName::Double => Ok(DataType::Float64), + r#gen::ColumnTypeName::Date => Ok(DataType::Date32), + r#gen::ColumnTypeName::Timestamp => Ok(DataType::Timestamp( TimeUnit::Microsecond, Some(Arc::from("UTC".to_string())), )), - types::ColumnTypeName::TimestampNtz => { + r#gen::ColumnTypeName::TimestampNtz => { Ok(DataType::Timestamp(TimeUnit::Microsecond, None)) } - types::ColumnTypeName::String | types::ColumnTypeName::Char => Ok(DataType::Utf8), - types::ColumnTypeName::Binary => Ok(DataType::Binary), - types::ColumnTypeName::Null => Ok(DataType::Null), - types::ColumnTypeName::Decimal => { + r#gen::ColumnTypeName::String | r#gen::ColumnTypeName::Char => Ok(DataType::Utf8), + r#gen::ColumnTypeName::Binary => Ok(DataType::Binary), + r#gen::ColumnTypeName::Null => Ok(DataType::Null), + r#gen::ColumnTypeName::Decimal => { if let (Some(precision), Some(scale)) = (type_precision, type_scale) { Ok(DataType::Decimal128(precision as u8, scale as i8)) } else if let Some(type_json) = type_json { @@ -521,7 +521,7 @@ pub(crate) fn unity_type_to_data_type( )) } } - types::ColumnTypeName::Interval => { + r#gen::ColumnTypeName::Interval => { if let Some(type_interval_type) = &type_interval_type { match type_interval_type.trim().to_uppercase().as_str() { "INTERVAL_YEAR_MONTH" @@ -548,16 +548,16 @@ pub(crate) fn unity_type_to_data_type( )) } } - types::ColumnTypeName::Array - | types::ColumnTypeName::Struct - | types::ColumnTypeName::Map => { + r#gen::ColumnTypeName::Array + | r#gen::ColumnTypeName::Struct + | r#gen::ColumnTypeName::Map => { parse_unity_type_json(type_json.as_deref().ok_or_else(|| { CatalogError::InvalidArgument( "type_json is required for complex types".to_string(), ) })?) } - types::ColumnTypeName::UserDefinedType | types::ColumnTypeName::TableType => Err( + r#gen::ColumnTypeName::UserDefinedType | r#gen::ColumnTypeName::TableType => Err( CatalogError::InvalidArgument(format!("{type_name:?} type is not supported yet",)), ), } @@ -746,7 +746,7 @@ pub(crate) fn parse_complex_type_from_json_object( _ => { return Err(CatalogError::InvalidArgument( "Invalid elementType format".to_string(), - )) + )); } }; Ok(DataType::List(Arc::new( @@ -787,7 +787,7 @@ pub(crate) fn parse_complex_type_from_json_object( _ => { return Err(CatalogError::InvalidArgument( "Invalid keyType format".to_string(), - )) + )); } }; let value_type = match value_type_json { @@ -796,7 +796,7 @@ pub(crate) fn parse_complex_type_from_json_object( _ => { return Err(CatalogError::InvalidArgument( "Invalid valueType format".to_string(), - )) + )); } }; @@ -854,7 +854,7 @@ pub(crate) fn parse_complex_type_from_json_object( _ => { return Err(CatalogError::InvalidArgument( "Invalid field type format".to_string(), - )) + )); } }; diff --git a/crates/sail-catalog-unity/src/lib.rs b/crates/sail-catalog-unity/src/lib.rs index ccb5fae74..0490288e7 100644 --- a/crates/sail-catalog-unity/src/lib.rs +++ b/crates/sail-catalog-unity/src/lib.rs @@ -15,9 +15,88 @@ mod data_type; mod provider; mod token; -pub mod unity { - #![expect(clippy::allow_attributes)] - include!(concat!(env!("OUT_DIR"), "/unity_catalog.rs")); +#[expect(unused_imports)] +#[expect(clippy::enum_variant_names)] +pub mod r#gen { + include!(concat!(env!("OUT_DIR"), "/unity_catalog_gen.rs")); + + // The column type name is defined manually to allow for custom aliases. + /// Name of type (INT, STRUCT, MAP, etc.). + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] + pub enum ColumnTypeName { + #[serde(rename = "BOOLEAN", alias = "boolean")] + Boolean, + #[serde(rename = "BYTE", alias = "byte")] + Byte, + #[serde(rename = "SHORT", alias = "short")] + Short, + #[serde(rename = "INT", alias = "int")] + Int, + #[serde(rename = "LONG", alias = "long")] + Long, + #[serde(rename = "FLOAT", alias = "float")] + Float, + #[serde(rename = "DOUBLE", alias = "double")] + Double, + #[serde(rename = "DATE", alias = "date")] + Date, + #[serde(rename = "TIMESTAMP", alias = "timestamp")] + Timestamp, + #[serde(rename = "TIMESTAMP_NTZ", alias = "timestamp_ntz")] + TimestampNtz, + #[serde(rename = "STRING", alias = "string")] + String, + #[serde(rename = "BINARY", alias = "binary")] + Binary, + #[serde(rename = "DECIMAL", alias = "decimal")] + Decimal, + #[serde(rename = "INTERVAL", alias = "interval")] + Interval, + #[serde(rename = "ARRAY", alias = "array")] + Array, + #[serde(rename = "STRUCT", alias = "struct")] + Struct, + #[serde(rename = "MAP", alias = "map")] + Map, + #[serde(rename = "CHAR", alias = "char")] + Char, + #[serde(rename = "NULL", alias = "null")] + Null, + #[serde(rename = "USER_DEFINED_TYPE", alias = "user_defined_type")] + UserDefinedType, + #[serde(rename = "TABLE_TYPE", alias = "table_type")] + TableType, + } + + impl std::fmt::Display for ColumnTypeName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let value = match self { + Self::Boolean => "BOOLEAN", + Self::Byte => "BYTE", + Self::Short => "SHORT", + Self::Int => "INT", + Self::Long => "LONG", + Self::Float => "FLOAT", + Self::Double => "DOUBLE", + Self::Date => "DATE", + Self::Timestamp => "TIMESTAMP", + Self::TimestampNtz => "TIMESTAMP_NTZ", + Self::String => "STRING", + Self::Binary => "BINARY", + Self::Decimal => "DECIMAL", + Self::Interval => "INTERVAL", + Self::Array => "ARRAY", + Self::Struct => "STRUCT", + Self::Map => "MAP", + Self::Char => "CHAR", + Self::Null => "NULL", + Self::UserDefinedType => "USER_DEFINED_TYPE", + Self::TableType => "TABLE_TYPE", + }; + f.write_str(value) + } + } } -pub use provider::UnityCatalogProvider; +pub use config::UnityCatalogConfig; +pub use provider::{UnityCatalogOptions, UnityCatalogProvider}; diff --git a/crates/sail-catalog-unity/src/provider.rs b/crates/sail-catalog-unity/src/provider.rs index 0da87b956..5020389b8 100644 --- a/crates/sail-catalog-unity/src/provider.rs +++ b/crates/sail-catalog-unity/src/provider.rs @@ -11,10 +11,10 @@ // limitations under the License. use std::collections::HashMap; -use std::str::FromStr; +use std::sync::Arc; use arrow::datatypes::DataType; -use reqwest::header::HeaderValue; +use sail_catalog::credentials::CatalogCredentials; use sail_catalog::error::{CatalogError, CatalogObject, CatalogResult}; use sail_catalog::lakehouse::{ DeltaRatifiedCommit, DeltaRatifiedCommitRequest, DeltaRatifiedCommitResponse, @@ -26,104 +26,77 @@ use sail_catalog::provider::{ CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropViewOptions, Namespace, TableFormatCreateMetadataMode, }; -use sail_catalog::utils::{get_property, quote_namespace_if_needed}; +use sail_catalog::utils::{get_property, quote_name_if_needed, quote_namespace_if_needed}; +use sail_common::utils::http::SAIL_USER_AGENT; use sail_common_datafusion::catalog::delta::{ - unity_table_id_value, DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, + DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, unity_table_id_value, }; use sail_common_datafusion::catalog::{ - identity_partition_fields, DatabaseStatus, TableColumnStatus, TableKind, TableStatus, + DatabaseStatus, TableColumnStatus, TableKind, TableStatus, identity_partition_fields, }; -use secrecy::SecretString; -use tokio::sync::OnceCell; -use crate::config::UnityCatalogConfig; -use crate::credential::CredentialProvider; use crate::data_type::{ data_type_to_unity_type, unity_struct_field_type_json, unity_type_to_data_type, }; -use crate::unity::{types, Client}; +use crate::r#gen; +use crate::r#gen::ApiClient; pub(crate) const DEFAULT_URI: &str = "http://localhost:8080/api/2.1/unity-catalog"; /// Provider for Unity Catalog pub struct UnityCatalogProvider { name: String, - catalog_config: UnityCatalogConfig, - client: OnceCell, + options: UnityCatalogOptions, + http_client: reqwest::Client, } -impl UnityCatalogProvider { - pub fn new( - name: String, - default_catalog: &Option, - uri: &Option, - token: &Option, - ) -> CatalogResult { - Self::with_options(name, default_catalog.clone(), uri.clone(), token, None) - } +#[derive(Debug, Clone)] +pub struct UnityCatalogOptions { + pub default_catalog: String, + pub uri: String, + pub credentials: Arc, + pub quote_object_name: bool, +} - pub fn with_options( - name: String, - default_catalog: Option, - uri: Option, - token: &Option, - options: Option>, - ) -> CatalogResult { - let catalog_config = UnityCatalogConfig::new(default_catalog, uri, token, options)?; +impl UnityCatalogProvider { + pub fn new(name: String, options: UnityCatalogOptions) -> CatalogResult { + let http_client = reqwest::Client::new(); Ok(Self { name, - catalog_config, - client: OnceCell::new(), + options, + http_client, }) } - async fn get_client(&self) -> CatalogResult<&Client> { - let config = &self.catalog_config; - let client = self - .client - .get_or_try_init(|| async move { - let mut client_builder = reqwest::Client::builder(); - - if let Some(credential_provider) = &config.credential_provider { - let header = match credential_provider { - CredentialProvider::BearerToken(token) => { - HeaderValue::from_str(&format!("Bearer {token}")).map_err(|e| { - CatalogError::External(format!( - "Failed to create header value from token: {e}" - )) - }) - } - CredentialProvider::TokenCredential(cache, cred) => { - let oauth_client = reqwest::Client::builder().build().map_err(|e| { - CatalogError::External(format!("Failed to build OAuth client: {e}")) - })?; - let token = cache - .get_or_insert_with(|| cred.fetch_token(&oauth_client)) - .await?; - HeaderValue::from_str(&format!("Bearer {token}")).map_err(|e| { - CatalogError::External(format!( - "Failed to create header value from token: {e}" - )) - }) - } - }?; - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert(reqwest::header::AUTHORIZATION, header); - client_builder = client_builder.default_headers(headers); - } - - if config.allow_http_url { - client_builder = client_builder.https_only(false); - } + async fn get_client(&self) -> CatalogResult { + let mut headers = reqwest::header::HeaderMap::new(); - let reqwest_client = client_builder.build().map_err(|e| { - CatalogError::External(format!("Failed to build HTTP client: {e}")) + if let Some(token) = self.options.credentials.retrieve().await? { + let header = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|e| { + CatalogError::External(format!("Failed to create header value from token: {e}")) })?; + headers.insert(reqwest::header::AUTHORIZATION, header); + } - Ok::<_, CatalogError>(Client::new_with_client(&config.uri, reqwest_client)) - }) - .await?; - Ok(client) + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static(SAIL_USER_AGENT), + ); + + Ok(ApiClient::new( + self.options.uri.clone(), + self.http_client.clone(), + headers, + )) + } + + fn object_name(&self, name: &str) -> String { + if self.options.quote_object_name { + quote_name_if_needed(name) + } else { + name.to_string() + } } fn get_catalog_and_schema_name( @@ -132,7 +105,7 @@ impl UnityCatalogProvider { ) -> CatalogResult<(String, String)> { match namespace.tail.as_slice() { [] => Ok(( - self.catalog_config.default_catalog.to_string(), + self.options.default_catalog.to_string(), namespace.head.to_string(), )), [x] => Ok((namespace.head.to_string(), x.to_string())), @@ -143,20 +116,44 @@ impl UnityCatalogProvider { } } - // Unity Catalog does not quote names in full names even if they contain special characters, - // so we only concatenate the names with dot ('.'). - - fn get_full_schema_name(catalog_name: &str, schema_name: &str) -> String { - format!("{catalog_name}.{schema_name}") + async fn create_managed_staging_location( + &self, + client: &ApiClient, + catalog_name: &str, + schema_name: &str, + table: &str, + ) -> CatalogResult { + let request = r#gen::CreateStagingTable { + name: self.object_name(table), + catalog_name: self.object_name(catalog_name), + schema_name: self.object_name(schema_name), + }; + let response = match client.create_staging_table(request).await { + Ok(response) => response, + Err(e) => { + return Err(CatalogError::External(format!( + "Failed to create staging table: {e}" + ))); + } + }; + response.inner.staging_location.ok_or_else(|| { + CatalogError::External( + "Unity Catalog staging table response is missing a staging location".to_string(), + ) + }) } - fn get_full_table_name(catalog_name: &str, schema_name: &str, table_name: &str) -> String { - format!("{catalog_name}.{schema_name}.{table_name}") + fn qualified_object_name(&self, names: &[&str]) -> String { + names + .iter() + .map(|name| self.object_name(name)) + .collect::>() + .join(".") } fn schema_info_to_database_status( &self, - schema_info: types::SchemaInfo, + schema_info: r#gen::SchemaInfo, catalog_name: &str, schema_name: Option<&str>, ) -> DatabaseStatus { @@ -168,37 +165,39 @@ impl UnityCatalogProvider { }; let database = vec![catalog_name.to_string(), schema_name]; - let mut properties: HashMap = - schema_info.properties.map(|p| p.0).unwrap_or_default(); - if let Some(created_at) = schema_info.created_at { - if !properties.contains_key("created_at") { - properties.insert("created_at".to_string(), created_at.to_string()); - } + let mut properties: HashMap = schema_info + .properties + .map(|p| p.0.into_iter().collect()) + .unwrap_or_default(); + if let Some(created_at) = schema_info.created_at + && !properties.contains_key("created_at") + { + properties.insert("created_at".to_string(), created_at.to_string()); } - if let Some(created_by) = schema_info.created_by { - if !properties.contains_key("created_by") { - properties.insert("created_by".to_string(), created_by); - } + if let Some(created_by) = schema_info.created_by + && !properties.contains_key("created_by") + { + properties.insert("created_by".to_string(), created_by); } - if let Some(owner) = schema_info.owner { - if !properties.contains_key("owner") { - properties.insert("owner".to_string(), owner); - } + if let Some(owner) = schema_info.owner + && !properties.contains_key("owner") + { + properties.insert("owner".to_string(), owner); } - if let Some(schema_id) = schema_info.schema_id { - if !properties.contains_key("schema_id") { - properties.insert("schema_id".to_string(), schema_id); - } + if let Some(schema_id) = schema_info.schema_id + && !properties.contains_key("schema_id") + { + properties.insert("schema_id".to_string(), schema_id); } - if let Some(updated_at) = schema_info.updated_at { - if !properties.contains_key("updated_at") { - properties.insert("updated_at".to_string(), updated_at.to_string()); - } + if let Some(updated_at) = schema_info.updated_at + && !properties.contains_key("updated_at") + { + properties.insert("updated_at".to_string(), updated_at.to_string()); } - if let Some(updated_by) = schema_info.updated_by { - if !properties.contains_key("updated_by") { - properties.insert("updated_by".to_string(), updated_by); - } + if let Some(updated_by) = schema_info.updated_by + && !properties.contains_key("updated_by") + { + properties.insert("updated_by".to_string(), updated_by); } DatabaseStatus { @@ -212,11 +211,11 @@ impl UnityCatalogProvider { fn table_info_to_table_status( &self, - table_info: types::TableInfo, + table_info: r#gen::TableInfo, catalog_name: &str, schema_name: &str, ) -> CatalogResult { - let types::TableInfo { + let r#gen::TableInfo { catalog_name: table_catalog_name, columns, comment, @@ -241,10 +240,15 @@ impl UnityCatalogProvider { let database = vec![catalog.clone(), schema]; let mut partition_by: Vec = vec![]; + // If all partition indices are 0, treat the table as unpartitioned. + // This can happen in the Unity Catalog endpoint in OneLake. + let columns = columns.unwrap_or_default(); + let ignore_partition_index = + !columns.is_empty() && columns.iter().all(|col| col.partition_index == Some(0)); let columns = columns .into_iter() .map(|col| { - let types::ColumnInfo { + let r#gen::ColumnInfo { comment, name, nullable, @@ -257,22 +261,27 @@ impl UnityCatalogProvider { type_scale, type_text, } = col; - if partition_index.is_some() { - if let Some(col_name) = &name { - partition_by.push(col_name.to_string()); - } + let partition_index = if ignore_partition_index { + None + } else { + partition_index + }; + if partition_index.is_some() + && let Some(col_name) = &name + { + partition_by.push(col_name.to_string()); } Ok(TableColumnStatus { name: name.unwrap_or_default(), data_type: unity_type_to_data_type( - type_name, + type_name.map(|type_name| *type_name), type_json, type_text, type_precision, type_scale, type_interval_type, )?, - nullable, + nullable: nullable.unwrap_or(true), comment, default: None, generated_always_as: None, @@ -288,7 +297,9 @@ impl UnityCatalogProvider { .map(|f| f.to_string().to_lowercase()) .unwrap_or_else(|| "delta".to_string()); - let mut properties: HashMap = properties.map(|p| p.0).unwrap_or_default(); + let mut properties: HashMap = properties + .map(|p| p.0.into_iter().collect()) + .unwrap_or_default(); let comment = comment.or(get_property(&properties, "comment")); @@ -316,7 +327,7 @@ impl UnityCatalogProvider { } let is_external = table_type .as_ref() - .is_none_or(|table_type| matches!(table_type, types::TableType::External)); + .is_none_or(|table_type| matches!(&**table_type, r#gen::TableType::External)); if let Some(table_type) = table_type { properties.insert("table_type".to_string(), table_type.to_string()); } @@ -376,30 +387,32 @@ impl CatalogProvider for UnityCatalogProvider { let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let request = types::CreateSchema::builder() - .catalog_name(&catalog_name) - .name(&schema_name) - .comment(comment) - .properties(if props.is_empty() { + let request = r#gen::CreateSchema { + catalog_name: self.object_name(&catalog_name), + name: self.object_name(&schema_name), + comment, + properties: if props.is_empty() { None } else { - Some(types::SecurablePropertiesMap::from(props)) - }); + Some(Box::new(r#gen::SecurablePropertiesMap( + props.into_iter().collect(), + ))) + }, + storage_root: None, + }; - let result = client.create_schema().body(request).send().await; + let result = client.create_schema(request).await; match result { Ok(response) => { - let schema_info = response.into_inner(); + let schema_info = response.inner; Ok(self.schema_info_to_database_status( schema_info, &catalog_name, Some(&schema_name), )) } - Err(progenitor_client::Error::UnexpectedResponse(response)) - if response.status().as_u16() == 409 && if_not_exists => - { + Err(e) if e.status() == Some(reqwest::StatusCode::CONFLICT) && if_not_exists => { self.get_database(database).await } Err(e) => Err(CatalogError::External(format!( @@ -415,22 +428,20 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_schema_name(&catalog_name, &schema_name); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name]); - let result = client.get_schema().full_name(&full_name).send().await; + let result = client.get_schema(full_name.clone()).await; match result { Ok(response) => { - let schema_info = response.into_inner(); + let schema_info = response.inner; Ok(self.schema_info_to_database_status( schema_info, &catalog_name, Some(&schema_name), )) } - Err(progenitor_client::Error::UnexpectedResponse(response)) - if response.status().as_u16() == 404 => - { + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => { Err(CatalogError::NotFound(CatalogObject::Schema, full_name)) } Err(e) => Err(CatalogError::External(format!("Failed to get schema: {e}"))), @@ -447,26 +458,25 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let catalog_name = match prefix { - None => self.catalog_config.default_catalog.to_string(), + None => self.options.default_catalog.to_string(), Some(Namespace { head, tail }) if tail.is_empty() => head.to_string(), Some(x) => { return Err(CatalogError::InvalidArgument(format!( "invalid prefix: {}", quote_namespace_if_needed(x) - ))) + ))); } }; let result = client - .list_schemas() - .catalog_name(&catalog_name) - .send() + .list_schemas(self.object_name(&catalog_name), None, None) .await; match result { Ok(response) => { - let list_response = response.into_inner(); + let list_response = response.inner; Ok(list_response .schemas + .unwrap_or_default() .into_iter() .map(|schema_info| { self.schema_info_to_database_status(schema_info, &catalog_name, None) @@ -492,27 +502,13 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_schema_name(&catalog_name, &schema_name); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name]); - let result = client - .delete_schema() - .full_name(full_name) - .force(cascade) - .send() - .await; + let result = client.delete_schema(full_name, Some(cascade)).await; match result { Ok(_) => Ok(()), - Err(progenitor_client::Error::InvalidResponsePayload(bytes, _)) - if bytes.as_ref() == b"200 OK" => - { - Ok(()) - } - Err(progenitor_client::Error::UnexpectedResponse(response)) - if response.status().as_u16() == 404 && if_exists => - { - Ok(()) - } + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) && if_exists => Ok(()), Err(e) => Err(CatalogError::External(format!( "Failed to drop schema: {e}" ))), @@ -537,7 +533,7 @@ impl CatalogProvider for UnityCatalogProvider { mode, properties, is_external, - is_write_precondition: _, + is_write_precondition, } = options; if mode.is_replace() { @@ -584,12 +580,9 @@ impl CatalogProvider for UnityCatalogProvider { } } - let data_source_format = types::DataSourceFormat::from_str(&format.trim().to_uppercase()) - .map_err(|e| { - CatalogError::InvalidArgument(format!("Invalid data source format: {e}")) - })?; + let data_source_format = parse_data_source_format(&format)?; - let unity_columns: Vec = columns + let unity_columns: Vec = columns .iter() .enumerate() .map(|(idx, col)| { @@ -621,10 +614,10 @@ impl CatalogProvider for UnityCatalogProvider { .iter() .position(|p| p.column.trim().to_lowercase() == col.name.trim().to_lowercase()) .map(|i| i as i32); - Ok(types::ColumnInfo { + Ok(r#gen::ColumnInfo { comment: col.comment.clone(), name: Some(col.name.clone()), - nullable: col.nullable, + nullable: Some(col.nullable), partition_index, position: Some(idx as i32), type_interval_type, @@ -637,7 +630,7 @@ impl CatalogProvider for UnityCatalogProvider { )? .to_string(), ), - type_name: Some(unity_type.type_name), + type_name: Some(Box::new(unity_type.type_name)), type_precision, type_scale, type_text: Some(unity_type.type_text), @@ -654,9 +647,9 @@ impl CatalogProvider for UnityCatalogProvider { } let table_type = if is_external { - types::TableType::External + r#gen::TableType::External } else { - types::TableType::Managed + r#gen::TableType::Managed }; let storage_location = if is_external { location.ok_or_else(|| { @@ -664,69 +657,63 @@ impl CatalogProvider for UnityCatalogProvider { "Storage location is required for external Unity Catalog tables".to_string(), ) })? - } else { - // The SQL planner supplies generated default locations for managed tables. Unity - // managed table creation must use a staging location allocated by the catalog. - let request = types::CreateStagingTable::builder() - .name(table) - .catalog_name(&catalog_name) - .schema_name(&schema_name); - let response = client - .create_staging_table() - .body(request) - .send() - .await - .map_err(|e| { - CatalogError::External(format!("Failed to create staging table: {e}")) - })?; - response.into_inner().staging_location.ok_or_else(|| { + } else if is_write_precondition { + // CTAS/data-write preconditions run before the writer, so they must use + // the staging location already returned by plan_lakehouse_create. + location.ok_or_else(|| { CatalogError::External( - "Unity Catalog staging table response is missing a staging location" + "Unity Catalog managed table write precondition is missing a planned staging location" .to_string(), ) })? + } else { + // The SQL planner supplies generated default locations for managed tables. Unity + // managed table creation must use a staging location allocated by the catalog. + self.create_managed_staging_location(&client, &catalog_name, &schema_name, table) + .await? }; - let request = types::CreateTable::builder() - .name(table) - .catalog_name(&catalog_name) - .schema_name(&schema_name) - .table_type(table_type) - .data_source_format(data_source_format) - .columns(unity_columns) - .storage_location(storage_location) - .comment(comment) - .properties(if props.is_empty() { + let request = r#gen::CreateTable { + name: self.object_name(table), + catalog_name: self.object_name(&catalog_name), + schema_name: self.object_name(&schema_name), + table_type: Box::new(table_type), + data_source_format: Box::new(data_source_format), + columns: unity_columns, + storage_location, + comment, + properties: if props.is_empty() { None } else { - Some(types::SecurablePropertiesMap::from(props)) - }); + Some(Box::new(r#gen::SecurablePropertiesMap( + props.into_iter().collect(), + ))) + }, + }; - let result = client.create_table().body(request).send().await; + let result = client.create_table(request).await; match result { Ok(response) => { - let table_info = response.into_inner(); + let table_info = response.inner; self.table_info_to_table_status(table_info, &catalog_name, &schema_name) } - Err(progenitor_client::Error::UnexpectedResponse(response)) - if response.status().as_u16() == 409 && mode.ignore_if_exists() => + Err(e) + if e.status() == Some(reqwest::StatusCode::CONFLICT) && mode.ignore_if_exists() => { self.get_table(database, table).await } - Err(progenitor_client::Error::UnexpectedResponse(response)) => { - let status = response.status(); - let body = response - .text() - .await - .unwrap_or_else(|e| format!("")); - Err(CatalogError::External(format!( - "Failed to create table: HTTP {status}: {body}" - ))) + Err(e) => { + if let Some(status) = e.status() { + Err(CatalogError::External(format!( + "Failed to create table: HTTP {status}: {e}" + ))) + } else { + Err(CatalogError::External(format!( + "Failed to create table: {e}" + ))) + } } - Err(e) => Err(CatalogError::External(format!( - "Failed to create table: {e}" - ))), } } @@ -737,18 +724,16 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_table_name(&catalog_name, &schema_name, table); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name, table]); - let result = client.get_table().full_name(&full_name).send().await; + let result = client.get_table(full_name.clone(), None, None).await; match result { Ok(response) => { - let table_info = response.into_inner(); + let table_info = response.inner; self.table_info_to_table_status(table_info, &catalog_name, &schema_name) } - Err(progenitor_client::Error::UnexpectedResponse(response)) - if response.status().as_u16() == 404 => - { + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => { Err(CatalogError::NotFound(CatalogObject::Table, full_name)) } Err(e) => Err(CatalogError::External(format!("Failed to get table: {e}"))), @@ -764,17 +749,20 @@ impl CatalogProvider for UnityCatalogProvider { let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; let result = client - .list_tables() - .catalog_name(&catalog_name) - .schema_name(&schema_name) - .send() + .list_tables( + self.object_name(&catalog_name), + self.object_name(&schema_name), + None, + None, + ) .await; match result { Ok(response) => { - let list_response = response.into_inner(); + let list_response = response.inner; list_response .tables + .unwrap_or_default() .into_iter() .map(|table_info| { self.table_info_to_table_status(table_info, &catalog_name, &schema_name) @@ -807,23 +795,11 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load config: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_table_name(&catalog_name, &schema_name, table); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name, table]); - match client.delete_table().full_name(full_name).send().await { + match client.delete_table(full_name).await { Ok(_) => Ok(()), - // The OSS Unity Catalog server currently returns a plain "200 OK" body - // for DELETE even though the OpenAPI spec declares a JSON response. - Err(progenitor_client::Error::InvalidResponsePayload(bytes, _)) - if bytes.as_ref() == b"200 OK" => - { - Ok(()) - } - Err(e) - if e.status() - .is_some_and(|status| status.as_u16() == 404 && if_exists) => - { - Ok(()) - } + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) && if_exists => Ok(()), Err(e) => Err(CatalogError::External(format!("Failed to drop table: {e}"))), } } @@ -862,11 +838,22 @@ impl CatalogProvider for UnityCatalogProvider { sail_catalog::provider::CreateTableMetadataRequirement::None, &self.lakehouse_capabilities(), ); - let _ = (database, table); if request.options.format.eq_ignore_ascii_case("delta") && !request.options.is_external { plan.materialization = LakehouseCreateMaterialization::AfterCatalogTableFormat { mode: TableFormatCreateMetadataMode::CatalogCoordinated, }; + if request.options.is_write_precondition { + let client = self + .get_client() + .await + .map_err(|e| CatalogError::External(format!("Failed to load client: {e}")))?; + let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; + let location = self + .create_managed_staging_location(&client, &catalog_name, &schema_name, table) + .await?; + plan.table.status.location = Some(location.clone()); + plan.table.execution.table_identity.table_uri = Some(location); + } } Ok(plan) } @@ -900,7 +887,7 @@ impl CatalogProvider for UnityCatalogProvider { "Unity Catalog Delta commit expects exactly one update payload".to_string(), )); }; - let commit: types::DeltaCommit = serde_json::from_value(update.clone()).map_err(|e| { + let commit: r#gen::DeltaCommit = serde_json::from_value(update.clone()).map_err(|e| { CatalogError::InvalidArgument(format!("Invalid Delta commit payload: {e}")) })?; @@ -910,34 +897,33 @@ impl CatalogProvider for UnityCatalogProvider { .map_err(|e| CatalogError::External(format!("Failed to load client: {e}")))?; let _ = (database, table); - match client.commit().body(commit).send().await { + match client.commit(commit).await { Ok(_) => Ok(LakehouseCommitOutcome::Committed { context, payload }), // The OSS Unity Catalog server currently returns a plain "200 OK" body // for Delta commits even though the OpenAPI spec declares a JSON response. - Err(progenitor_client::Error::InvalidResponsePayload(bytes, _)) - if bytes.as_ref() == b"200 OK" => - { + // Such responses are considered unknown errors since JSON deserialization fails. + Err(e) if e.status().is_some_and(|s| s.is_success()) => { Ok(LakehouseCommitOutcome::Committed { context, payload }) } - Err(e) if e.status().is_some_and(|status| status.as_u16() == 409) => Err( + Err(e) if e.status() == Some(reqwest::StatusCode::CONFLICT) => Err( CatalogError::Conflict(format!("Unity Catalog Delta commit conflict: {e}")), ), - Err(e) if e.status().is_some_and(|status| status.as_u16() == 400) => { + Err(e) if e.status() == Some(reqwest::StatusCode::BAD_REQUEST) => { Err(CatalogError::InvalidArgument(format!( "Unity Catalog Delta commit invalid argument: {e}" ))) } - Err(e) if e.status().is_some_and(|status| status.as_u16() == 401) => Err( + Err(e) if e.status() == Some(reqwest::StatusCode::UNAUTHORIZED) => Err( CatalogError::Unauthorized(format!("Unity Catalog Delta commit unauthorized: {e}")), ), - Err(e) if e.status().is_some_and(|status| status.as_u16() == 403) => Err( + Err(e) if e.status() == Some(reqwest::StatusCode::FORBIDDEN) => Err( CatalogError::Forbidden(format!("Unity Catalog Delta commit forbidden: {e}")), ), - Err(e) if e.status().is_some_and(|status| status.as_u16() == 429) => Err( + Err(e) if e.status() == Some(reqwest::StatusCode::TOO_MANY_REQUESTS) => Err( CatalogError::RateLimited(format!("Unity Catalog Delta commit rate limited: {e}")), ), - Err(e) if e.status().is_some_and(|status| status.as_u16() == 501) => Err( - CatalogError::NotSupported("Unity Catalog Delta commit endpoint".to_string()), + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_IMPLEMENTED) => Err( + CatalogError::NotSupported(format!("Unity Catalog Delta commit endpoint: {e}")), ), Err(e) => Err(CatalogError::External(format!( "Failed to commit Delta table to Unity Catalog: {e}" @@ -992,7 +978,7 @@ impl CatalogProvider for UnityCatalogProvider { )) })?; - let request = types::DeltaGetCommits { + let request = r#gen::DeltaGetCommits { end_version: request.end_version, start_version: request.start_version, table_id: table_id.to_string(), @@ -1003,11 +989,11 @@ impl CatalogProvider for UnityCatalogProvider { .await .map_err(|e| CatalogError::External(format!("Failed to load client: {e}")))?; let (catalog_name, schema_name) = self.get_catalog_and_schema_name(database)?; - let full_name = Self::get_full_table_name(&catalog_name, &schema_name, table); + let full_name = self.qualified_object_name(&[&catalog_name, &schema_name, table]); - match client.get_commits().body(request).send().await { + match client.get_commits(request).await { Ok(response) => { - let response = response.into_inner(); + let response = response.inner; Ok(DeltaRatifiedCommitResponse { latest_table_version: response.latest_table_version, commits: response @@ -1023,18 +1009,18 @@ impl CatalogProvider for UnityCatalogProvider { .collect(), }) } - Err(e) if e.status().is_some_and(|status| status.as_u16() == 400) => { + Err(e) if e.status() == Some(reqwest::StatusCode::BAD_REQUEST) => { Err(CatalogError::InvalidArgument(format!( "Unity Catalog Delta commit discovery invalid argument: {e}" ))) } - Err(e) if e.status().is_some_and(|status| status.as_u16() == 404) => { + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_FOUND) => { Err(CatalogError::NotFound(CatalogObject::Table, full_name)) } - Err(e) if e.status().is_some_and(|status| status.as_u16() == 501) => { - Err(CatalogError::NotSupported( - "Unity Catalog Delta commit discovery endpoint".to_string(), - )) + Err(e) if e.status() == Some(reqwest::StatusCode::NOT_IMPLEMENTED) => { + Err(CatalogError::NotSupported(format!( + "Unity Catalog Delta commit discovery endpoint: {e}" + ))) } Err(e) => Err(CatalogError::External(format!( "Failed to get Delta commits from Unity Catalog: {e}" @@ -1077,12 +1063,58 @@ impl CatalogProvider for UnityCatalogProvider { } } +fn parse_data_source_format(format: &str) -> CatalogResult { + match format.trim().to_ascii_uppercase().as_str() { + "DELTA" => Ok(r#gen::DataSourceFormat::Delta), + "CSV" => Ok(r#gen::DataSourceFormat::Csv), + "JSON" => Ok(r#gen::DataSourceFormat::Json), + "AVRO" => Ok(r#gen::DataSourceFormat::Avro), + "PARQUET" => Ok(r#gen::DataSourceFormat::Parquet), + "ORC" => Ok(r#gen::DataSourceFormat::Orc), + "TEXT" => Ok(r#gen::DataSourceFormat::Text), + "ICEBERG" => Ok(r#gen::DataSourceFormat::Iceberg), + other => Err(CatalogError::InvalidArgument(format!( + "Invalid data source format: {other}" + ))), + } +} + #[expect(clippy::unwrap_used)] #[cfg(test)] mod tests { + use sail_catalog::credentials::{CatalogCredentials, EmptyCatalogCredentials}; use secrecy::ExposeSecret; use super::*; + use crate::config::UnityCatalogConfig; + + fn options_from_config( + default_catalog: impl Into, + config: UnityCatalogConfig, + ) -> UnityCatalogOptions { + let credentials = config + .get_credential_provider() + .map(|credentials| Arc::new(credentials) as Arc) + .unwrap_or_else(|| Arc::new(EmptyCatalogCredentials)); + UnityCatalogOptions { + default_catalog: default_catalog.into(), + uri: config.uri, + credentials, + quote_object_name: true, + } + } + + fn empty_options( + default_catalog: impl Into, + quote_object_name: bool, + ) -> UnityCatalogOptions { + UnityCatalogOptions { + default_catalog: default_catalog.into(), + uri: DEFAULT_URI.to_string(), + credentials: Arc::new(EmptyCatalogCredentials), + quote_object_name, + } + } #[test] fn test_config_with_token() { @@ -1093,7 +1125,7 @@ mod tests { "https://test.databricks.com".to_string(), ); - let config = UnityCatalogConfig::new(None, None, &None, Some(options)).unwrap(); + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); assert!(config.bearer_token.is_some()); assert_eq!(config.bearer_token.unwrap().expose_secret(), "test_token"); assert_eq!(config.uri, "https://test.databricks.com"); @@ -1119,7 +1151,7 @@ mod tests { "test_tenant".to_string(), ); - let config = UnityCatalogConfig::new(None, None, &None, Some(options)).unwrap(); + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); assert_eq!(config.uri, "https://test.databricks.com"); assert_eq!(config.client_id, Some("test_client_id".to_string())); assert!(config.client_secret.is_some()); @@ -1142,7 +1174,7 @@ mod tests { options.insert("unity_allow_http_url".to_string(), value.to_string()); options.insert("unity_use_azure_cli".to_string(), value.to_string()); - let config = UnityCatalogConfig::new(None, None, &None, Some(options)).unwrap(); + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); assert_eq!(config.allow_http_url, expected, "Failed for value: {value}"); assert_eq!(config.use_azure_cli, expected, "Failed for value: {value}"); } @@ -1166,7 +1198,7 @@ mod tests { let test_value = format!("test_value_for_{key}"); options.insert(key.to_string(), test_value.clone()); - let result = UnityCatalogConfig::new(None, None, &None, Some(options)); + let result = UnityCatalogConfig::new(None, &None, Some(options)); assert!(result.is_ok(), "Failed to parse key: {key}"); let config = result.unwrap(); @@ -1188,18 +1220,197 @@ mod tests { "https://test.databricks.com".to_string(), ); - let provider = UnityCatalogProvider::with_options( + let config = UnityCatalogConfig::new(None, &None, Some(options)).unwrap(); + assert!(config.bearer_token.is_some()); + + let provider = UnityCatalogProvider::new( "test".to_string(), - Some("test_catalog".to_string()), - None, - &None, - Some(options), + options_from_config("test_catalog", config), ) .unwrap(); assert_eq!(provider.name, "test"); - assert_eq!(provider.catalog_config.default_catalog, "test_catalog"); - assert_eq!(provider.catalog_config.uri, "https://test.databricks.com"); - assert!(provider.catalog_config.bearer_token.is_some()); + assert_eq!(provider.options.default_catalog, "test_catalog"); + assert_eq!(provider.options.uri, "https://test.databricks.com"); + } + + #[test] + fn test_provider_quotes_object_names_by_default() { + let config = UnityCatalogConfig::new(None, &None, None).unwrap(); + let provider = UnityCatalogProvider::new( + "test".to_string(), + options_from_config("catalog.name", config), + ) + .unwrap(); + + assert_eq!(provider.options.default_catalog, "catalog.name"); + assert_eq!( + provider.object_name(&provider.options.default_catalog), + "`catalog.name`" + ); + assert_eq!(provider.object_name("schema.name"), "`schema.name`"); + assert_eq!( + provider.qualified_object_name(&["catalog.name", "schema.name", "table.name"]), + "`catalog.name`.`schema.name`.`table.name`" + ); + let namespace = Namespace::try_from(vec!["schema.name"]).unwrap(); + assert_eq!( + provider.get_catalog_and_schema_name(&namespace).unwrap(), + ("catalog.name".to_string(), "schema.name".to_string()) + ); + } + + #[test] + fn test_provider_can_preserve_unquoted_object_names() { + let provider = UnityCatalogProvider::new( + "test".to_string(), + empty_options("lakehouse.Lakehouse", false), + ) + .unwrap(); + + assert_eq!(provider.options.default_catalog, "lakehouse.Lakehouse"); + assert_eq!( + provider.object_name(&provider.options.default_catalog), + "lakehouse.Lakehouse" + ); + assert_eq!(provider.object_name("schema.name"), "schema.name"); + assert_eq!( + provider.qualified_object_name(&["catalog.name", "schema.name", "table.name"]), + "catalog.name.schema.name.table.name" + ); + } + + #[test] + #[expect(clippy::panic)] + fn test_table_info_accepts_null_columns_and_lowercase_column_type_name() { + let response: r#gen::ListTablesResponse = serde_json::from_value(serde_json::json!({ + "tables": [{ + "catalog_name": "catalog", + "schema_name": "schema", + "name": "empty_table", + "columns": null + }, { + "catalog_name": "catalog", + "schema_name": "schema", + "name": "typed_table", + "columns": [{ + "name": "value", + "type_name": "string", + "type_text": "string", + "nullable": true + }] + }] + })) + .unwrap(); + + let provider = + UnityCatalogProvider::new("test".to_string(), empty_options("catalog", true)).unwrap(); + + let mut tables = response.tables.unwrap_or_default().into_iter(); + let empty_table = provider + .table_info_to_table_status(tables.next().unwrap(), "catalog", "schema") + .unwrap(); + let typed_table = provider + .table_info_to_table_status(tables.next().unwrap(), "catalog", "schema") + .unwrap(); + + let TableKind::Table { + columns, comment, .. + } = empty_table.kind + else { + panic!("expected table status"); + }; + assert_eq!(comment, None); + assert!(columns.is_empty()); + + let TableKind::Table { columns, .. } = typed_table.kind else { + panic!("expected table status"); + }; + assert_eq!(columns[0].data_type, DataType::Utf8); + } + + #[test] + #[expect(clippy::panic)] + fn test_table_info_ignores_onelake_all_zero_partition_indexes() { + let response: r#gen::TableInfo = serde_json::from_value(serde_json::json!({ + "catalog_name": "catalog", + "schema_name": "schema", + "name": "typed_table", + "columns": [{ + "name": "id", + "type_name": "int", + "type_text": "int", + "nullable": true, + "partition_index": 0 + }, { + "name": "value", + "type_name": "string", + "type_text": "string", + "nullable": true, + "partition_index": 0 + }] + })) + .unwrap(); + + let provider = + UnityCatalogProvider::new("test".to_string(), empty_options("catalog", true)).unwrap(); + + let status = provider + .table_info_to_table_status(response, "catalog", "schema") + .unwrap(); + + let TableKind::Table { + columns, + partition_by, + .. + } = status.kind + else { + panic!("expected table status"); + }; + assert!(partition_by.is_empty()); + assert!(columns.iter().all(|column| !column.is_partition)); + } + + #[test] + #[expect(clippy::panic)] + fn test_table_info_preserves_real_partition_index() { + let response: r#gen::TableInfo = serde_json::from_value(serde_json::json!({ + "catalog_name": "catalog", + "schema_name": "schema", + "name": "typed_table", + "columns": [{ + "name": "id", + "type_name": "int", + "type_text": "int", + "nullable": true + }, { + "name": "value", + "type_name": "string", + "type_text": "string", + "nullable": true, + "partition_index": 0 + }] + })) + .unwrap(); + + let provider = + UnityCatalogProvider::new("test".to_string(), empty_options("catalog", true)).unwrap(); + + let status = provider + .table_info_to_table_status(response, "catalog", "schema") + .unwrap(); + + let TableKind::Table { + columns, + partition_by, + .. + } = status.kind + else { + panic!("expected table status"); + }; + assert_eq!(partition_by.len(), 1); + assert_eq!(partition_by[0].column, "value"); + assert!(!columns[0].is_partition); + assert!(columns[1].is_partition); } } diff --git a/crates/sail-catalog-unity/tests/rest_integration_test.rs b/crates/sail-catalog-unity/tests/rest_integration_test.rs index 7af8c226f..d46e4c8a8 100644 --- a/crates/sail-catalog-unity/tests/rest_integration_test.rs +++ b/crates/sail-catalog-unity/tests/rest_integration_test.rs @@ -16,13 +16,14 @@ use std::collections::HashMap; use std::sync::Arc; use arrow::datatypes::{DataType, Field, Fields}; +use sail_catalog::credentials::EmptyCatalogCredentials; use sail_catalog::provider::{ CatalogPartitionField, CatalogProvider, CreateDatabaseOptions, CreateTableColumnOptions, CreateTableMode, CreateTableOptions, DropDatabaseOptions, DropTableOptions, Namespace, RuntimeAwareCatalogProvider, }; -use sail_catalog_unity::unity::{types, Client}; -use sail_catalog_unity::UnityCatalogProvider; +use sail_catalog_unity::r#gen::{self, ApiClient}; +use sail_catalog_unity::{UnityCatalogOptions, UnityCatalogProvider}; use sail_common::runtime::RuntimeHandle; use sail_common::spec::{ SAIL_LIST_FIELD_NAME, SAIL_MAP_FIELD_NAME, SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME, @@ -72,7 +73,7 @@ async fn setup_catalog( RuntimeAwareCatalogProvider, ContainerAsync, ContainerAsync, - Client, + ApiClient, ) { let network = format!("unity_{}", network_name); @@ -145,25 +146,31 @@ async fn setup_catalog( || { UnityCatalogProvider::new( "sail".to_string(), - &Some(DEFAULT_CATALOG.to_string()), - &Some(rest_url.clone()), - &None, + UnityCatalogOptions { + default_catalog: DEFAULT_CATALOG.to_string(), + uri: rest_url.clone(), + credentials: Arc::new(EmptyCatalogCredentials), + quote_object_name: true, + }, ) }, runtime.io().clone(), ) .expect("Failed to create runtime-aware catalog"); - let client = Client::new(&rest_url).unwrap(); + let client = ApiClient::new( + rest_url.clone(), + reqwest::Client::new(), + reqwest::header::HeaderMap::new(), + ); for attempt in 1..=5 { match client - .create_catalog() - .body( - types::CreateCatalog::builder() - .name(DEFAULT_CATALOG) - .comment(Some("Main catalog for testing".to_string())), - ) - .send() + .create_catalog(r#gen::CreateCatalog { + name: DEFAULT_CATALOG.to_string(), + comment: Some("Main catalog for testing".to_string()), + properties: None, + storage_root: None, + }) .await { Ok(_) => break, @@ -225,15 +232,21 @@ async fn test_create_schema() { assert_eq!(properties.len(), 6); assert_eq!(static_properties, expected_properties); - assert!(properties - .iter() - .any(|(k, v)| k == "schema_id" && !v.is_empty())); - assert!(properties - .iter() - .any(|(k, v)| k == "updated_at" && !v.is_empty())); - assert!(properties - .iter() - .any(|(k, v)| k == "created_at" && !v.is_empty())); + assert!( + properties + .iter() + .any(|(k, v)| k == "schema_id" && !v.is_empty()) + ); + assert!( + properties + .iter() + .any(|(k, v)| k == "updated_at" && !v.is_empty()) + ); + assert!( + properties + .iter() + .any(|(k, v)| k == "created_at" && !v.is_empty()) + ); assert_eq!(catalog, "sail".to_string()); assert_eq!(database, Vec::::from(full_namespace.clone())); @@ -361,28 +374,34 @@ async fn test_get_schema() { Vec::::from(full_namespace.clone()) ); for (key, value) in &properties { - assert!(created_db - .properties - .iter() - .any(|(k, v)| k == key && v == value)); + assert!( + created_db + .properties + .iter() + .any(|(k, v)| k == key && v == value) + ); } let get_db = unity_catalog.get_database(&namespace).await.unwrap(); assert_eq!(get_db.database, Vec::::from(full_namespace.clone())); for (key, value) in &properties { - assert!(get_db - .properties - .iter() - .any(|(k, v)| k == key && v == value)); + assert!( + get_db + .properties + .iter() + .any(|(k, v)| k == key && v == value) + ); } let get_db = unity_catalog.get_database(&full_namespace).await.unwrap(); assert_eq!(get_db.database, Vec::::from(full_namespace)); for (key, value) in &properties { - assert!(get_db - .properties - .iter() - .any(|(k, v)| k == key && v == value)); + assert!( + get_db + .properties + .iter() + .any(|(k, v)| k == key && v == value) + ); } } @@ -411,11 +430,13 @@ async fn test_list_schemas() { let parent = Namespace::try_from(vec![DEFAULT_CATALOG.to_string()]).unwrap(); assert!(unity_catalog.list_databases(None).await.unwrap().is_empty()); - assert!(unity_catalog - .list_databases(Some(&parent)) - .await - .unwrap() - .is_empty()); + assert!( + unity_catalog + .list_databases(Some(&parent)) + .await + .unwrap() + .is_empty() + ); unity_catalog .create_database( @@ -445,21 +466,25 @@ async fn test_list_schemas() { let dbs = unity_catalog.list_databases(Some(&parent)).await.unwrap(); assert_eq!(dbs.len(), 2); - assert!(dbs - .iter() - .any(|db| db.database == Vec::::from(ns1_full.clone()))); - assert!(dbs - .iter() - .any(|db| db.database == Vec::::from(ns2_full.clone()))); + assert!( + dbs.iter() + .any(|db| db.database == Vec::::from(ns1_full.clone())) + ); + assert!( + dbs.iter() + .any(|db| db.database == Vec::::from(ns2_full.clone())) + ); let dbs = unity_catalog.list_databases(None).await.unwrap(); assert_eq!(dbs.len(), 2); - assert!(dbs - .iter() - .any(|db| db.database == Vec::::from(ns1_full.clone()))); - assert!(dbs - .iter() - .any(|db| db.database == Vec::::from(ns2_full.clone()))); + assert!( + dbs.iter() + .any(|db| db.database == Vec::::from(ns1_full.clone())) + ); + assert!( + dbs.iter() + .any(|db| db.database == Vec::::from(ns2_full.clone())) + ); } #[tokio::test] @@ -944,12 +969,16 @@ async fn test_create_table() { assert!(properties.contains(&("option.key1".to_string(), "value1".to_string()))); assert!(properties.contains(&("owner".to_string(), "mr. meow".to_string()))); assert!(properties.contains(&("team".to_string(), "data-eng".to_string()))); - assert!(properties - .iter() - .any(|(key, _)| key == DELTA_UNITY_TABLE_ID_LEGACY_KEY)); - assert!(properties - .iter() - .any(|(key, _)| key == DELTA_UNITY_TABLE_ID_KEY)); + assert!( + properties + .iter() + .any(|(key, _)| key == DELTA_UNITY_TABLE_ID_LEGACY_KEY) + ); + assert!( + properties + .iter() + .any(|(key, _)| key == DELTA_UNITY_TABLE_ID_KEY) + ); assert_eq!(columns.len(), 3); assert!( columns.contains(&sail_common_datafusion::catalog::TableColumnStatus { diff --git a/crates/sail-catalog/Cargo.toml b/crates/sail-catalog/Cargo.toml index 1e16a7528..0345f61ab 100644 --- a/crates/sail-catalog/Cargo.toml +++ b/crates/sail-catalog/Cargo.toml @@ -23,3 +23,6 @@ log = { workspace = true } sail-common-datafusion = { path = "../sail-common-datafusion" } sail-common = { path = "../sail-common" } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/sail-catalog/src/command.rs b/crates/sail-catalog/src/command.rs index f2b049d77..ba99cf2df 100644 --- a/crates/sail-catalog/src/command.rs +++ b/crates/sail-catalog/src/command.rs @@ -1,10 +1,10 @@ use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef; use sail_common_datafusion::array::serde::ArrowSerializer; -use sail_common_datafusion::catalog::LakehouseOperation; +use sail_common_datafusion::catalog::{FunctionStatus, LakehouseOperation}; use sail_common_datafusion::datasource::{ - is_lakehouse_format, TableFormatAlterTableOperation, TableFormatCreateTableColumn, - TableFormatCreateTableInfo, TableFormatRegistry, + TableFormatAlterTableOperation, TableFormatCreateTableColumn, TableFormatCreateTableInfo, + TableFormatRegistry, is_lakehouse_format, }; use sail_common_datafusion::extension::SessionExtensionAccessor; use sail_common_datafusion::session::plan::PlanService; @@ -14,8 +14,8 @@ use crate::error::{CatalogError, CatalogObject, CatalogResult}; use crate::lakehouse::{ LakehouseCreateMaterialization, LakehouseCreatePlan, LakehouseCreateRequest, }; -use crate::manager::tracker::{CatalogFunctionId, CatalogLogicalPlanId}; use crate::manager::CatalogManager; +use crate::manager::tracker::{CatalogFunctionId, CatalogLogicalPlanId}; use crate::provider::{ AlterTableOptions, CreateDatabaseOptions, CreateTableOptions, CreateTemporaryViewOptions, CreateViewOptions, DropDatabaseOptions, DropTableOptions, DropTemporaryViewOptions, @@ -72,6 +72,13 @@ pub enum CatalogCommand { database: Vec, pattern: String, }, + ShowFunctions { + database: Vec, + pattern: Option, + system_functions: Vec, + show_user_functions: bool, + show_system_functions: bool, + }, ListTables { database: Vec, pattern: Option, @@ -101,6 +108,12 @@ pub enum CatalogCommand { ListFunctions { database: Vec, pattern: Option, + system_functions: Vec, + }, + DescribeFunction { + function: Vec, + extended: bool, + system_functions: Vec, }, DropFunction { function: Vec, @@ -168,6 +181,7 @@ impl CatalogCommand { CatalogCommand::GetTable { .. } => "GetTable", CatalogCommand::ShowTables { .. } => "ShowTables", CatalogCommand::ShowTableExtended { .. } => "ShowTableExtended", + CatalogCommand::ShowFunctions { .. } => "ShowFunctions", CatalogCommand::ListTables { .. } => "ListTables", CatalogCommand::ListViews { .. } => "ListViews", CatalogCommand::DropTable { .. } => "DropTable", @@ -176,6 +190,7 @@ impl CatalogCommand { CatalogCommand::FunctionExists { .. } => "FunctionExists", CatalogCommand::GetFunction { .. } => "GetFunction", CatalogCommand::ListFunctions { .. } => "ListFunctions", + CatalogCommand::DescribeFunction { .. } => "DescribeFunction", CatalogCommand::RegisterFunction { .. } => "RegisterFunction", CatalogCommand::RegisterTableFunction { .. } => "RegisterTableFunction", CatalogCommand::DropFunction { .. } => "DropFunction", @@ -205,10 +220,16 @@ impl CatalogCommand { CatalogCommand::ShowTableExtended { .. } => { ArrowSerializer::default().schema::()? } + CatalogCommand::ShowFunctions { .. } => { + ArrowSerializer::default().schema::()? + } CatalogCommand::ListColumns { .. } => display.table_columns().schema()?, CatalogCommand::GetFunction { .. } | CatalogCommand::ListFunctions { .. } => { display.functions().schema()? } + CatalogCommand::DescribeFunction { .. } => { + ArrowSerializer::default().schema::()? + } CatalogCommand::SetCurrentCatalog { .. } | CatalogCommand::SetCurrentDatabase { .. } | CatalogCommand::RegisterFunction { .. } @@ -378,6 +399,29 @@ impl CatalogCommand { .collect::>>()?; ArrowSerializer::default().build_record_batch(&rows)? } + CatalogCommand::ShowFunctions { + database, + pattern, + system_functions, + show_user_functions, + show_system_functions, + } => { + let rows = manager + .list_functions( + &database, + pattern.as_deref(), + &system_functions, + show_user_functions, + show_system_functions, + ) + .await? + .into_iter() + .map(|status| ShowFunctionsRow { + function: status.name, + }) + .collect::>(); + ArrowSerializer::default().build_record_batch(&rows)? + } CatalogCommand::ListTables { database, pattern } => { let rows = manager .list_tables_and_views(&database, pattern.as_deref()) @@ -546,8 +590,41 @@ impl CatalogCommand { CatalogCommand::GetFunction { .. } => { return Err(CatalogError::NotSupported("get function".to_string())); } - CatalogCommand::ListFunctions { .. } => { - return Err(CatalogError::NotSupported("list functions".to_string())); + CatalogCommand::ListFunctions { + database, + pattern, + system_functions, + } => { + let rows = manager + .list_functions(&database, pattern.as_deref(), &system_functions, true, true) + .await?; + display.functions().to_record_batch(rows)? + } + CatalogCommand::DescribeFunction { + function, + extended, + system_functions, + } => { + let status = manager + .get_function_status(&function, &system_functions) + .await?; + let mut rows = vec![DescribeFunctionRow { + function_desc: format!("Function: {}", status.name), + }]; + if !status.class_name.is_empty() { + rows.push(DescribeFunctionRow { + function_desc: format!("Class: {}", status.class_name), + }); + } + rows.push(DescribeFunctionRow { + function_desc: format!("Usage: {}", status.usage()), + }); + if extended { + rows.push(DescribeFunctionRow { + function_desc: format!("Extended Usage:{}", status.extended_usage()), + }); + } + ArrowSerializer::default().build_record_batch(&rows)? } CatalogCommand::DropFunction { function, @@ -988,6 +1065,16 @@ struct ShowTableExtendedRow { information: String, } +#[derive(Serialize, Deserialize)] +struct ShowFunctionsRow { + function: String, +} + +#[derive(Serialize, Deserialize)] +struct DescribeFunctionRow { + function_desc: String, +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -1000,7 +1087,7 @@ mod tests { use datafusion_expr::{LogicalPlan, TableSource}; use sail_common_datafusion::catalog::display::{CatalogObjectDisplay, DefaultCatalogDisplay}; use sail_common_datafusion::catalog::{ - DatabaseStatus, TableColumnStatus, TableKind, TableStatus, + DatabaseStatus, FunctionStatus, TableColumnStatus, TableKind, TableStatus, }; use sail_common_datafusion::datasource::{SinkInfo, SourceInfo, TableFormat}; use sail_common_datafusion::session::plan::{PlanFormatter, PlanService}; @@ -1042,8 +1129,8 @@ mod tests { TestCatalogOutput { value: status.name } } - fn function(name: String) -> Self::Function { - TestCatalogOutput { value: name } + fn function(status: FunctionStatus) -> Self::Function { + TestCatalogOutput { value: status.name } } } diff --git a/crates/sail-catalog/src/credentials.rs b/crates/sail-catalog/src/credentials.rs new file mode 100644 index 000000000..80991b548 --- /dev/null +++ b/crates/sail-catalog/src/credentials.rs @@ -0,0 +1,176 @@ +use std::fmt::Debug; +use std::path::PathBuf; + +use crate::error::{CatalogError, CatalogResult}; + +#[async_trait::async_trait] +pub trait CatalogCredentials: Debug + Send + Sync + 'static { + async fn retrieve(&self) -> CatalogResult>; +} + +#[derive(Debug, Default)] +pub struct EmptyCatalogCredentials; + +#[async_trait::async_trait] +impl CatalogCredentials for EmptyCatalogCredentials { + async fn retrieve(&self) -> CatalogResult> { + Ok(None) + } +} + +#[derive(Debug)] +pub struct StaticCatalogCredentials { + credential: String, +} + +impl StaticCatalogCredentials { + pub fn new(credential: String) -> Self { + Self { credential } + } +} + +#[async_trait::async_trait] +impl CatalogCredentials for StaticCatalogCredentials { + async fn retrieve(&self) -> CatalogResult> { + Ok(Some(self.credential.clone())) + } +} + +/// Credentials backed by a token file on disk, such as a kubelet-projected +/// service account token. The file is read on every call, so a rotated token +/// is picked up without restarting the server. The Iceberg REST provider reads +/// the credential fresh for each request and retries once on a `401`, so a +/// token that rotates mid-operation is recovered without an in-memory cache. +#[derive(Debug)] +pub struct FileCatalogCredentials { + path: PathBuf, +} + +impl FileCatalogCredentials { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } +} + +#[async_trait::async_trait] +impl CatalogCredentials for FileCatalogCredentials { + async fn retrieve(&self) -> CatalogResult> { + let credential = tokio::fs::read_to_string(&self.path) + .await + .map_err(|e| { + CatalogError::External(format!( + "failed to read token file {}: {e}", + self.path.display() + )) + })? + .trim() + .to_string(); + if credential.is_empty() { + return Err(CatalogError::External(format!( + "token file {} is empty", + self.path.display() + ))); + } + Ok(Some(credential)) + } +} + +#[cfg(test)] +mod tests { + #![expect(clippy::unwrap_used)] + + use std::fs::File; + use std::io::Write; + use std::path::Path; + + use tempfile::TempDir; + + use super::*; + + fn write_token(path: &Path, contents: &str) { + let mut file = File::create(path).unwrap(); + file.write_all(contents.as_bytes()).unwrap(); + } + + #[tokio::test] + async fn retrieve_returns_token_from_file() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("token"); + write_token(&path, "s3cr3t-token"); + + let credentials = FileCatalogCredentials::new(&path); + assert_eq!( + credentials.retrieve().await.unwrap(), + Some("s3cr3t-token".to_string()) + ); + } + + #[tokio::test] + async fn retrieve_trims_surrounding_whitespace() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("token"); + write_token(&path, " s3cr3t-token\n\n"); + + let credentials = FileCatalogCredentials::new(&path); + assert_eq!( + credentials.retrieve().await.unwrap(), + Some("s3cr3t-token".to_string()) + ); + } + + #[tokio::test] + async fn retrieve_rereads_rotated_token() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("token"); + write_token(&path, "first-token"); + + let credentials = FileCatalogCredentials::new(&path); + assert_eq!( + credentials.retrieve().await.unwrap(), + Some("first-token".to_string()) + ); + + // Every call reads the file, so a rotated token (for example kubelet + // swapping a projected service account token) is picked up on the next + // retrieve without restarting the server. + write_token(&path, "second-token"); + assert_eq!( + credentials.retrieve().await.unwrap(), + Some("second-token".to_string()) + ); + } + + #[tokio::test] + async fn retrieve_reports_error_for_missing_file() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("does-not-exist"); + + let credentials = FileCatalogCredentials::new(&path); + let error = credentials.retrieve().await.unwrap_err(); + assert!( + matches!(error, CatalogError::External(_)), + "unexpected error variant: {error:?}" + ); + } + + #[tokio::test] + async fn retrieve_reports_error_for_empty_file() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("token"); + write_token(&path, "\n \n"); + + let credentials = FileCatalogCredentials::new(&path); + let error = credentials.retrieve().await.unwrap_err(); + assert!( + matches!(&error, CatalogError::External(message) if message.contains("empty")), + "unexpected error: {error:?}" + ); + + // Once the file holds a token again, the next retrieve returns it. + write_token(&path, "recovered-token"); + assert_eq!( + credentials.retrieve().await.unwrap(), + Some("recovered-token".to_string()) + ); + } +} diff --git a/crates/sail-catalog/src/hive_format.rs b/crates/sail-catalog/src/hive_format.rs index 6a3ea31c9..edfa5b442 100644 --- a/crates/sail-catalog/src/hive_format.rs +++ b/crates/sail-catalog/src/hive_format.rs @@ -80,7 +80,7 @@ pub struct HiveCatalogFormat { impl HiveCatalogFormat { pub fn from_format(format: &str) -> CatalogResult { match format.trim().to_ascii_lowercase().as_str() { - "delta" | "deltalake" => Ok(Self { + "delta" => Ok(Self { logical_format: "delta", storage_format: HiveStorageFormat::parquet(), }), @@ -179,7 +179,6 @@ mod tests { #![expect(clippy::unwrap_used)] use super::{HiveCatalogFormat, HiveDetectedFormat, HiveStorageFormat}; - #[test] fn test_textfile_storage_format_uses_lazy_simple_serde() { let format = HiveStorageFormat::textfile(); @@ -229,7 +228,7 @@ mod tests { #[test] fn test_catalog_format_maps_delta_to_parquet_storage() { - let format = HiveCatalogFormat::from_format("deltalake").unwrap(); + let format = HiveCatalogFormat::from_format("delta").unwrap(); assert_eq!(format.logical_format, "delta"); assert_eq!(format.storage_format, HiveStorageFormat::parquet()); } diff --git a/crates/sail-catalog/src/lakehouse.rs b/crates/sail-catalog/src/lakehouse.rs index 4de7fd904..7be8de6b8 100644 --- a/crates/sail-catalog/src/lakehouse.rs +++ b/crates/sail-catalog/src/lakehouse.rs @@ -1,5 +1,5 @@ use sail_common_datafusion::catalog::delta::{ - unity_table_id_value, DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, + DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, unity_table_id_value, }; use sail_common_datafusion::catalog::{ CapabilityFingerprint, CatalogProviderId, CatalogTableIdentity, CommitAuthority, diff --git a/crates/sail-catalog/src/lib.rs b/crates/sail-catalog/src/lib.rs index d129e1572..bcb97e8f9 100644 --- a/crates/sail-catalog/src/lib.rs +++ b/crates/sail-catalog/src/lib.rs @@ -1,4 +1,5 @@ pub mod command; +pub mod credentials; pub mod error; pub mod hive_format; pub mod lakehouse; diff --git a/crates/sail-catalog/src/manager/function.rs b/crates/sail-catalog/src/manager/function.rs index 9b866c5e8..26a7b7df4 100644 --- a/crates/sail-catalog/src/manager/function.rs +++ b/crates/sail-catalog/src/manager/function.rs @@ -2,10 +2,12 @@ use std::sync::Arc; use datafusion::catalog::TableFunctionImpl; use datafusion_expr::ScalarUDF; +use sail_common_datafusion::catalog::FunctionStatus; use crate::command::CatalogTableFunction; use crate::error::{CatalogError, CatalogObject, CatalogResult}; use crate::manager::CatalogManager; +use crate::utils::match_pattern; impl CatalogManager { fn canonical_function_name(name: &str) -> Arc { @@ -25,6 +27,70 @@ impl CatalogManager { Ok(state.functions.get(&name).cloned()) } + pub async fn list_functions>( + &self, + database: &[T], + pattern: Option<&str>, + system_functions: &[FunctionStatus], + show_user_functions: bool, + show_system_functions: bool, + ) -> CatalogResult> { + let _ = self.get_database_by_qualifier(database).await?; + let state = self.state()?; + let mut functions = if show_system_functions { + system_functions + .iter() + .filter(|status| match_pattern(&status.name, pattern)) + .cloned() + .collect::>() + } else { + vec![] + }; + if show_user_functions { + functions.extend( + state + .functions + .keys() + .filter(|name| match_pattern(name.as_ref(), pattern)) + .map(|name| FunctionStatus::temporary(name.to_string())), + ); + } + functions.sort_by(|left, right| left.name.cmp(&right.name)); + functions.dedup_by(|left, right| left.name == right.name); + Ok(functions) + } + + pub async fn get_function_status>( + &self, + function: &[T], + system_functions: &[FunctionStatus], + ) -> CatalogResult { + let (name, database) = function + .split_last() + .ok_or_else(|| CatalogError::InvalidArgument("empty function name".to_string()))?; + if !database.is_empty() { + let _ = self.get_database_by_qualifier(database).await?; + return Err(CatalogError::NotFound( + CatalogObject::Function, + function + .iter() + .map(|part| part.as_ref()) + .collect::>() + .join("."), + )); + } + let name = Self::canonical_function_name(name.as_ref()); + let state = self.state()?; + if state.functions.contains_key(&name) { + return Ok(FunctionStatus::temporary(name.to_string())); + } + system_functions + .iter() + .find(|status| Self::canonical_function_name(&status.name) == name) + .cloned() + .ok_or_else(|| CatalogError::NotFound(CatalogObject::Function, name.to_string())) + } + pub fn register_table_function( &self, _name: String, diff --git a/crates/sail-catalog/src/manager/table.rs b/crates/sail-catalog/src/manager/table.rs index aaf047dcc..1e149d448 100644 --- a/crates/sail-catalog/src/manager/table.rs +++ b/crates/sail-catalog/src/manager/table.rs @@ -2,11 +2,10 @@ use sail_common_datafusion::catalog::TableStatus; use crate::error::{CatalogError, CatalogObject, CatalogResult}; use crate::lakehouse::{ - resolve_lakehouse_table_status, BeginTableAccessRequest, DeltaRatifiedCommitRequest, - DeltaRatifiedCommitResponse, LakehouseCommitOutcome, LakehouseCommitRequest, - LakehouseCreatePlan, LakehouseCreateRequest, LakehouseResolvedTable, - LakehouseScanPlanningRequest, LakehouseScanPlanningResponse, ResolveLakehouseTableRequest, - TableAccessSession, + BeginTableAccessRequest, DeltaRatifiedCommitRequest, DeltaRatifiedCommitResponse, + LakehouseCommitOutcome, LakehouseCommitRequest, LakehouseCreatePlan, LakehouseCreateRequest, + LakehouseResolvedTable, LakehouseScanPlanningRequest, LakehouseScanPlanningResponse, + ResolveLakehouseTableRequest, TableAccessSession, resolve_lakehouse_table_status, }; use crate::manager::CatalogManager; use crate::provider::{ @@ -139,7 +138,7 @@ impl CatalogManager { // list_views; treat that as "no views" so SHOW TABLES still works there. let views = match views_res { Ok(v) => v, - Err(CatalogError::NotSupported(_)) => vec![], + Err(CatalogError::NotSupported(_)) | Err(CatalogError::NotFound(_, _)) => vec![], Err(e) => return Err(e), }; tables.extend(views); @@ -202,10 +201,10 @@ impl CatalogManager { Err(e) => return Err(e), } } - if let [x @ .., name] = reference { - if self.state()?.is_global_temporary_view_database(x) { - return self.get_global_temporary_view(name.as_ref()).await; - } + if let [x @ .., name] = reference + && self.state()?.is_global_temporary_view_database(x) + { + return self.get_global_temporary_view(name.as_ref()).await; } match self.get_table(reference).await { Ok(x) => return Ok(x), diff --git a/crates/sail-catalog/src/manager/view.rs b/crates/sail-catalog/src/manager/view.rs index 8bff20b0c..f81e8ea96 100644 --- a/crates/sail-catalog/src/manager/view.rs +++ b/crates/sail-catalog/src/manager/view.rs @@ -126,13 +126,13 @@ impl CatalogManager { Err(e) => return Err(e), } } - if let [x @ .., name] = view { - if self.state()?.is_global_temporary_view_database(x) { - match GLOBAL_TEMPORARY_VIEW_MANAGER.drop_view(name.as_ref(), false) { - Ok(_) => return Ok(()), - Err(CatalogError::NotFound(_, _)) => {} - Err(e) => return Err(e), - } + if let [x @ .., name] = view + && self.state()?.is_global_temporary_view_database(x) + { + match GLOBAL_TEMPORARY_VIEW_MANAGER.drop_view(name.as_ref(), false) { + Ok(_) => return Ok(()), + Err(CatalogError::NotFound(_, _)) => {} + Err(e) => return Err(e), } } self.drop_view(view, options).await diff --git a/crates/sail-catalog/src/provider/cache.rs b/crates/sail-catalog/src/provider/cache.rs index 48466822f..90fd92c46 100644 --- a/crates/sail-catalog/src/provider/cache.rs +++ b/crates/sail-catalog/src/provider/cache.rs @@ -459,8 +459,8 @@ impl CatalogProvider for CachingCatalogPr #[cfg(test)] #[expect(clippy::unwrap_used)] mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; use sail_common_datafusion::catalog::{ CatalogProviderId, CatalogTableIdentity, CommitAuthority, DatabaseStatus, diff --git a/crates/sail-catalog/src/provider/mod.rs b/crates/sail-catalog/src/provider/mod.rs index 9784f1c25..fc62261df 100644 --- a/crates/sail-catalog/src/provider/mod.rs +++ b/crates/sail-catalog/src/provider/mod.rs @@ -11,11 +11,11 @@ use sail_common_datafusion::catalog::{DatabaseStatus, TableStatus}; use crate::error::{CatalogError, CatalogResult}; use crate::lakehouse::{ - plan_lakehouse_create_from_requirement, resolve_lakehouse_table_status, BeginTableAccessRequest, DeltaRatifiedCommitRequest, DeltaRatifiedCommitResponse, LakehouseCapability, LakehouseCommitOutcome, LakehouseCommitRequest, LakehouseCreatePlan, LakehouseCreateRequest, LakehouseResolvedTable, LakehouseScanPlanningRequest, LakehouseScanPlanningResponse, ResolveLakehouseTableRequest, TableAccessSession, + plan_lakehouse_create_from_requirement, resolve_lakehouse_table_status, }; /// A trait that defines the interface for a catalog. diff --git a/crates/sail-cli/src/flight/server.rs b/crates/sail-cli/src/flight/server.rs index f76b92968..9b2a19cca 100644 --- a/crates/sail-cli/src/flight/server.rs +++ b/crates/sail-cli/src/flight/server.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use log::info; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeManager; -use sail_telemetry::telemetry::{init_telemetry, shutdown_telemetry, ResourceOptions}; +use sail_telemetry::telemetry::{ResourceOptions, init_telemetry, shutdown_telemetry}; use tokio::net::TcpListener; async fn shutdown() { @@ -33,7 +33,7 @@ pub fn run_flight_server(ip: IpAddr, port: u16) -> Result<(), Box Result<(), Box> { // in the child process. It uses `sys.executable` to determine the location of // the Python interpreter. When the Sail CLI runs as a standalone binary, the // Python interpreter is embedded and `sys.executable` points to the Sail binary. - std::env::set_var(CliConfigEnv::RUN_PYTHON, "true"); + // SAFETY: This runs during CLI startup before Python initialization creates threads. + unsafe { std::env::set_var(CliConfigEnv::RUN_PYTHON, "true") }; // Initialize the Python interpreter. Python::initialize(); let args = std::env::args().collect(); diff --git a/crates/sail-cli/src/python.rs b/crates/sail-cli/src/python.rs index 3220aea3f..f0220d1fa 100644 --- a/crates/sail-cli/src/python.rs +++ b/crates/sail-cli/src/python.rs @@ -1,9 +1,9 @@ use std::ffi::CString; use std::marker::PhantomData; -use log::{error, logger, Level, MetadataBuilder, Record}; +use log::{Level, MetadataBuilder, Record, error, logger}; use pyo3::prelude::{PyAnyMethods, PyModule, PyModuleMethods}; -use pyo3::{pyclass, pymethods, Bound, PyAny, PyResult, Python}; +use pyo3::{Bound, PyAny, PyResult, Python, pyclass, pymethods}; pub struct Module { name: &'static str, @@ -101,7 +101,7 @@ impl NativeLogging { logger().log( &Record::builder() .metadata(metadata) - .args(format_args!("{}", &message)) + .args(format_args!("{}", message)) .line(Some(line)) .file(Some(&pathname)) .module_path(Some(&pathname)) diff --git a/crates/sail-cli/src/runner.rs b/crates/sail-cli/src/runner.rs index 28501d81f..106f5578e 100644 --- a/crates/sail-cli/src/runner.rs +++ b/crates/sail-cli/src/runner.rs @@ -4,7 +4,7 @@ use sail_common::error::CommonError; use crate::flight::run_flight_server; use crate::spark::run::run_pyspark_script; use crate::spark::{ - run_pyspark_shell, run_spark_connect_server, run_spark_mcp_server, McpSettings, McpTransport, + McpSettings, McpTransport, run_pyspark_shell, run_spark_connect_server, run_spark_mcp_server, }; use crate::worker::run_worker; diff --git a/crates/sail-cli/src/spark/mod.rs b/crates/sail-cli/src/spark/mod.rs index 3f9980d5b..878522df3 100644 --- a/crates/sail-cli/src/spark/mod.rs +++ b/crates/sail-cli/src/spark/mod.rs @@ -3,6 +3,6 @@ pub(crate) mod run; mod server; mod shell; -pub(crate) use mcp_server::{run_spark_mcp_server, McpSettings, McpTransport}; +pub(crate) use mcp_server::{McpSettings, McpTransport, run_spark_mcp_server}; pub(crate) use server::run_spark_connect_server; pub(crate) use shell::run_pyspark_shell; diff --git a/crates/sail-cli/src/spark/server.rs b/crates/sail-cli/src/spark/server.rs index c54ac4978..c2bdc55bd 100644 --- a/crates/sail-cli/src/spark/server.rs +++ b/crates/sail-cli/src/spark/server.rs @@ -28,7 +28,7 @@ async fn shutdown() { pub(super) mod telemetry { use sail_common::config::AppConfig; - use sail_telemetry::telemetry::{init_telemetry, shutdown_telemetry, ResourceOptions}; + use sail_telemetry::telemetry::{ResourceOptions, init_telemetry, shutdown_telemetry}; pub struct TelemetryGuard { /// A marker to prevent struct creation without calling [`TelemetryGuard::try_new()`]. diff --git a/crates/sail-cli/src/worker/entrypoint.rs b/crates/sail-cli/src/worker/entrypoint.rs index a734cd578..ee227b086 100644 --- a/crates/sail-cli/src/worker/entrypoint.rs +++ b/crates/sail-cli/src/worker/entrypoint.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeManager; use sail_session::session_factory::{SessionFactory, WorkerSessionFactory}; -use sail_telemetry::telemetry::{init_telemetry, shutdown_telemetry, ResourceOptions}; +use sail_telemetry::telemetry::{ResourceOptions, init_telemetry, shutdown_telemetry}; pub fn run_worker() -> Result<(), Box> { let config = Arc::new(AppConfig::load()?); diff --git a/crates/sail-common-datafusion/src/array/placeholder.rs b/crates/sail-common-datafusion/src/array/placeholder.rs index 7bbc608c8..325cb0549 100644 --- a/crates/sail-common-datafusion/src/array/placeholder.rs +++ b/crates/sail-common-datafusion/src/array/placeholder.rs @@ -1,21 +1,21 @@ use std::sync::Arc; use datafusion::arrow::array::{ - new_null_array, ArrayData, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, BinaryArray, - BinaryViewArray, BooleanBuilder, DictionaryArray, FixedSizeBinaryArray, FixedSizeListArray, - LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, ListArray, - ListViewArray, MapArray, NullArray, PrimitiveArray, PrimitiveBuilder, RunArray, StringArray, - StringViewArray, StructArray, UnionArray, + ArrayData, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, BinaryArray, BinaryViewArray, + BooleanBuilder, DictionaryArray, FixedSizeBinaryArray, FixedSizeListArray, LargeBinaryArray, + LargeListArray, LargeListViewArray, LargeStringArray, ListArray, ListViewArray, MapArray, + NullArray, PrimitiveArray, PrimitiveBuilder, RunArray, StringArray, StringViewArray, + StructArray, UnionArray, new_null_array, }; use datafusion::arrow::buffer::{Buffer, ScalarBuffer}; use datafusion::arrow::datatypes::{ - ArrowTimestampType, DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, - Decimal32Type, Decimal64Type, DurationMicrosecondType, DurationMillisecondType, - DurationNanosecondType, DurationSecondType, Float16Type, Float32Type, Float64Type, Int16Type, - Int32Type, Int64Type, Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, + ArrowTimestampType, DataType, Date32Type, Date64Type, Decimal32Type, Decimal64Type, + Decimal128Type, Decimal256Type, DurationMicrosecondType, DurationMillisecondType, + DurationNanosecondType, DurationSecondType, Float16Type, Float32Type, Float64Type, Int8Type, + Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, - TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, + TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionMode, }; use datafusion_common::internal_err; diff --git a/crates/sail-common-datafusion/src/array/record_batch.rs b/crates/sail-common-datafusion/src/array/record_batch.rs index 8a528766c..7b6ac25a8 100644 --- a/crates/sail-common-datafusion/src/array/record_batch.rs +++ b/crates/sail-common-datafusion/src/array/record_batch.rs @@ -2,8 +2,8 @@ use std::io::Cursor; use std::sync::Arc; use datafusion::arrow::array::{ - new_null_array, Array, ArrayRef, LargeListArray, ListArray, MapArray, PrimitiveArray, - RecordBatch, RecordBatchOptions, StructArray, + Array, ArrayRef, LargeListArray, ListArray, MapArray, PrimitiveArray, RecordBatch, + RecordBatchOptions, StructArray, new_null_array, }; use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::{ @@ -108,7 +108,7 @@ where )) } -fn cast_array_recursively(src: &ArrayRef, target_type: &DataType) -> Result { +pub fn cast_array_recursively(src: &ArrayRef, target_type: &DataType) -> Result { let src_type = src.data_type(); if src_type == target_type { return Ok(src.clone()); @@ -117,24 +117,23 @@ fn cast_array_recursively(src: &ArrayRef, target_type: &DataType) -> Result { - adjust_timestamp_timezone::(src, target_tz.clone())? - } - TimeUnit::Millisecond => { - adjust_timestamp_timezone::(src, target_tz.clone())? - } - TimeUnit::Microsecond => { - adjust_timestamp_timezone::(src, target_tz.clone())? - } - TimeUnit::Nanosecond => { - adjust_timestamp_timezone::(src, target_tz.clone())? - } - }; - return Ok(adjusted); - } + let adjusted = match src_unit { + TimeUnit::Second => { + adjust_timestamp_timezone::(src, target_tz.clone())? + } + TimeUnit::Millisecond => { + adjust_timestamp_timezone::(src, target_tz.clone())? + } + TimeUnit::Microsecond => { + adjust_timestamp_timezone::(src, target_tz.clone())? + } + TimeUnit::Nanosecond => { + adjust_timestamp_timezone::(src, target_tz.clone())? + } + }; + return Ok(adjusted); } match (src_type, target_type) { diff --git a/crates/sail-common-datafusion/src/array/serde.rs b/crates/sail-common-datafusion/src/array/serde.rs index 553c56b37..341514d62 100644 --- a/crates/sail-common-datafusion/src/array/serde.rs +++ b/crates/sail-common-datafusion/src/array/serde.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use datafusion::arrow::array::{RecordBatch, RecordBatchOptions}; use datafusion::arrow::datatypes::{FieldRef, Schema, SchemaRef}; -use datafusion_common::{internal_datafusion_err, Result}; +use datafusion_common::{Result, internal_datafusion_err}; use serde::{Deserialize, Serialize}; use serde_arrow::schema::{SchemaLike, TracingOptions}; use serde_arrow::to_arrow; diff --git a/crates/sail-common-datafusion/src/catalog/display.rs b/crates/sail-common-datafusion/src/catalog/display.rs index 6de9acacc..199959779 100644 --- a/crates/sail-common-datafusion/src/catalog/display.rs +++ b/crates/sail-common-datafusion/src/catalog/display.rs @@ -4,7 +4,7 @@ use datafusion::common::Result; use serde::{Deserialize, Serialize}; use crate::array::serde::ArrowSerializer; -use crate::catalog::{DatabaseStatus, TableColumnStatus, TableStatus}; +use crate::catalog::{DatabaseStatus, FunctionStatus, TableColumnStatus, TableStatus}; /// A trait for displaying catalog objects in a structured format. /// This is useful for defining output schemas and producing outputs @@ -24,7 +24,7 @@ pub trait CatalogObjectDisplay { fn table_column(status: TableColumnStatus) -> Self::TableColumn; - fn function(name: String) -> Self::Function; + fn function(status: FunctionStatus) -> Self::Function; } /// A trait for providing output display for catalog command. @@ -40,7 +40,7 @@ pub trait CatalogDisplay: Send + Sync { fn tables(&self) -> Box>; fn databases(&self) -> Box>; fn table_columns(&self) -> Box>; - fn functions(&self) -> Box>; + fn functions(&self) -> Box>; } pub trait OutputDisplay { @@ -126,8 +126,8 @@ where Box::new(>::new(D::table_column)) } - fn functions(&self) -> Box> { - Box::new(>::new( + fn functions(&self) -> Box> { + Box::new(>::new( D::function, )) } diff --git a/crates/sail-common-datafusion/src/catalog/status.rs b/crates/sail-common-datafusion/src/catalog/status.rs index f8fe7f995..d4028ce5f 100644 --- a/crates/sail-common-datafusion/src/catalog/status.rs +++ b/crates/sail-common-datafusion/src/catalog/status.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Fields}; -use datafusion_common::{plan_err, Result}; +use datafusion_common::{Result, plan_err}; use datafusion_expr::LogicalPlan; use serde::{Deserialize, Serialize}; @@ -34,6 +34,86 @@ pub struct TableStatus { pub kind: TableKind, } +#[derive(Debug, Clone, Eq, PartialEq, Hash, PartialOrd, Serialize, Deserialize)] +pub struct FunctionStatus { + pub name: String, + pub catalog: Option, + pub namespace: Option>, + pub signatures: Vec, + pub usage: Option, + pub arguments: Option, + pub examples: Option, + pub note: Option, + pub since: Option, + pub deprecated: Option, + pub class_name: String, + pub is_temporary: bool, +} + +impl FunctionStatus { + pub fn temporary(name: String) -> Self { + Self { + name, + catalog: None, + namespace: None, + signatures: vec![], + usage: None, + arguments: None, + examples: None, + note: None, + since: None, + deprecated: None, + class_name: String::new(), + is_temporary: true, + } + } + + pub fn list_description(&self) -> Option { + self.usage.clone() + } + + pub fn usage(&self) -> String { + self.usage.clone().unwrap_or_else(|| "N/A.".to_string()) + } + + pub fn extended_usage(&self) -> String { + let mut output = String::new(); + if let Some(arguments) = self.arguments.as_deref().filter(|value| !value.is_empty()) { + output.push_str(arguments); + } + if let Some(examples) = self.examples.as_deref().filter(|value| !value.is_empty()) { + output.push_str(examples); + } + if output.is_empty() { + output.push('\n'); + output.push_str(" No example/argument for "); + output.push_str(&self.name); + output.push_str(".\n"); + } + if let Some(note) = self.note.as_deref().filter(|value| !value.is_empty()) { + output.push('\n'); + output.push_str(" Note:\n"); + output.push_str(" "); + output.push_str(note.trim()); + output.push('\n'); + } + if let Some(since) = self.since.as_deref().filter(|value| !value.is_empty()) { + output.push('\n'); + output.push_str(" Since: "); + output.push_str(since); + output.push('\n'); + } + if let Some(deprecated) = self.deprecated.as_deref().filter(|value| !value.is_empty()) { + output.push('\n'); + output.push_str(" Deprecated:\n"); + output.push_str(" "); + output.push_str(deprecated.trim()); + output.push('\n'); + } + output + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum TableKind { Table { diff --git a/crates/sail-common-datafusion/src/datasource.rs b/crates/sail-common-datafusion/src/datasource.rs index ef854a979..50fcccb69 100644 --- a/crates/sail-common-datafusion/src/datasource.rs +++ b/crates/sail-common-datafusion/src/datasource.rs @@ -9,9 +9,9 @@ use datafusion::common::plan_datafusion_err; use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::logical_expr::LogicalPlan; use datafusion::physical_expr::{ - create_physical_sort_exprs, LexOrdering, LexRequirement, PhysicalSortRequirement, + LexOrdering, LexRequirement, PhysicalSortRequirement, create_physical_sort_exprs, }; -use datafusion_common::{not_impl_err, plan_err, Constraints, DFSchema, DFSchemaRef, Result}; +use datafusion_common::{Constraints, DFSchema, DFSchemaRef, Result, not_impl_err, plan_err}; use datafusion_expr::expr::Sort; use datafusion_expr::{Expr, TableSource}; @@ -66,7 +66,16 @@ impl OptionLayer { /// The returned map can be passed to code that accepts `HashMap`. pub fn into_opaque_options(self) -> HashMap { match self { - OptionLayer::TablePropertyList { items } => items.into_iter().collect(), + OptionLayer::TablePropertyList { items } => items + .into_iter() + .map(|(key, value)| { + if let Some(key) = key.strip_prefix("option.") { + (key.to_string(), value) + } else { + (key, value) + } + }) + .collect(), OptionLayer::OptionList { items } => items.into_iter().collect(), OptionLayer::TableLocation { .. } | OptionLayer::AsOfTimestamp { .. } @@ -304,6 +313,7 @@ pub struct MergeIntoOptions { pub source_alias: Option, pub target: MergeTargetInfo, pub with_schema_evolution: bool, + pub case_sensitive: bool, /// Resolved logical schemas from analysis time (before any rewrites) pub resolved_target_schema: DFSchemaRef, pub resolved_source_schema: DFSchemaRef, @@ -312,6 +322,8 @@ pub struct MergeIntoOptions { /// to real column names without the fragile recover-field-names heuristic. pub resolved_target_field_names: Vec, pub resolved_source_field_names: Vec, + /// User-facing source column name and its collision-free internal alias. + pub source_column_aliases: Vec<(String, String)>, pub on_condition: ExprWithSource, pub matched_clauses: Vec, pub not_matched_by_source_clauses: Vec, @@ -327,6 +339,8 @@ pub struct MergeIntoOptions { /// references target schema field IDs and is rewritten to actual column names /// by MERGE expansion before being applied as a post-processing projection. pub generated_column_exprs: Vec<(String, Expr)>, + /// Resolved target column defaults used by explicit MERGE inserts. + pub default_column_exprs: Vec<(String, Expr)>, /// Delta CHECK constraint expressions for the target table. pub check_constraint_exprs: Vec, } @@ -723,8 +737,8 @@ mod tests { use super::*; #[test] - fn missing_jdbc_table_format_error_includes_registration_hint( - ) -> std::result::Result<(), String> { + fn missing_jdbc_table_format_error_includes_registration_hint() + -> std::result::Result<(), String> { let registry = TableFormatRegistry::new(); let error = match registry.get("jdbc") { Ok(_) => return Err("expected missing jdbc table format error".to_string()), diff --git a/crates/sail-common-datafusion/src/display.rs b/crates/sail-common-datafusion/src/display.rs index 73d390c5c..b81eed146 100644 --- a/crates/sail-common-datafusion/src/display.rs +++ b/crates/sail-common-datafusion/src/display.rs @@ -6,14 +6,14 @@ use chrono::{Duration, NaiveDate, SecondsFormat, TimeZone, Utc}; use datafusion::arrow::array::timezone::Tz; use datafusion::arrow::array::*; use datafusion::arrow::datatypes::{ - ArrowDictionaryKeyType, ArrowNativeType, DataType, Date32Type, Date64Type, Decimal128Type, - Decimal256Type, Decimal32Type, Decimal64Type, DecimalType, DurationMicrosecondType, + ArrowDictionaryKeyType, ArrowNativeType, DataType, Date32Type, Date64Type, Decimal32Type, + Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType, DurationSecondType, Float16Type, Float32Type, - Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, IntervalDayTimeType, + Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalYearMonthType, RunEndIndexType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType, - TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, - UInt64Type, UInt8Type, UnionMode, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, + UInt32Type, UInt64Type, UnionMode, }; use datafusion::arrow::error::ArrowError; use datafusion::arrow::temporal_conversions::{ @@ -546,7 +546,7 @@ macro_rules! decimal_display { decimal_display!(Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type); macro_rules! try_convert { - ($value:expr, $f:expr, $t:expr $(,)?) => { + ($value:expr_2021, $f:expr_2021, $t:expr_2021 $(,)?) => { $f($value).ok_or_else(|| { ArrowError::CastError(format!(concat!("invalid ", $t, " value: {}"), $value)) }) @@ -554,7 +554,7 @@ macro_rules! try_convert { } macro_rules! timestamp_display { - ($t:ty, $formatter:expr $(,)?) => { + ($t:ty, $formatter:expr_2021 $(,)?) => { impl<'a> DisplayIndexState<'a> for &'a PrimitiveArray<$t> { type State = (Option, TimeFormat<'a>); diff --git a/crates/sail-common-datafusion/src/error.rs b/crates/sail-common-datafusion/src/error.rs index 5159ea6e9..a9c1a8174 100644 --- a/crates/sail-common-datafusion/src/error.rs +++ b/crates/sail-common-datafusion/src/error.rs @@ -1,16 +1,17 @@ use std::collections::HashSet; +use std::fmt; +use std::fmt::Formatter; use datafusion::arrow::error::ArrowError; use datafusion_common::DataFusionError; use serde::{Deserialize, Serialize}; use thiserror::Error; -/// A Python error in text form. This could be a remote error from a worker. +/// An error with a cause received from a remote worker. #[derive(Debug, Clone, Error)] -#[error("remote Python error: {summary}")] -pub struct RemotePythonError { - pub summary: String, - pub traceback: Option>, +#[error("remote error: {cause}")] +pub struct RemoteError { + pub cause: CommonErrorCause, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -19,24 +20,6 @@ pub struct PythonErrorCause { pub traceback: Option>, } -impl From for PythonErrorCause { - fn from(error: RemotePythonError) -> Self { - Self { - summary: error.summary, - traceback: error.traceback, - } - } -} - -impl From for RemotePythonError { - fn from(cause: PythonErrorCause) -> Self { - Self { - summary: cause.summary, - traceback: cause.traceback, - } - } -} - /// A trait to extract Python error cause from a generic error. /// /// The implementation should inspect the provided error without @@ -80,6 +63,41 @@ pub enum CommonErrorCause { DeltaTable(String), } +impl fmt::Display for CommonErrorCause { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + CommonErrorCause::Unknown(x) + | CommonErrorCause::Internal(x) + | CommonErrorCause::NotImplemented(x) + | CommonErrorCause::InvalidArgument(x) + | CommonErrorCause::Io(x) + | CommonErrorCause::ArrowCast(x) + | CommonErrorCause::ArrowMemory(x) + | CommonErrorCause::ArrowParse(x) + | CommonErrorCause::ArrowCompute(x) + | CommonErrorCause::ArrowIpc(x) + | CommonErrorCause::ArrowCDataInterface(x) + | CommonErrorCause::ArrowDivideByZero(x) + | CommonErrorCause::ArrowArithmeticOverflow(x) + | CommonErrorCause::ArrowDictionaryKeyOverflow(x) + | CommonErrorCause::ArrowRunEndIndexOverflow(x) + | CommonErrorCause::ArrowOffsetOverflow(x) + | CommonErrorCause::FormatCsv(x) + | CommonErrorCause::FormatJson(x) + | CommonErrorCause::FormatParquet(x) + | CommonErrorCause::FormatAvro(x) + | CommonErrorCause::Plan(x) + | CommonErrorCause::Schema(x) + | CommonErrorCause::Configuration(x) + | CommonErrorCause::Execution(x) + | CommonErrorCause::DeltaTable(x) => write!(f, "{x}"), + CommonErrorCause::Python(PythonErrorCause { summary, .. }) => { + write!(f, "{summary}") + } + } + } +} + impl CommonErrorCause { fn build( error: &(dyn std::error::Error + 'static), @@ -152,8 +170,8 @@ impl CommonErrorCause { return Self::Python(cause); } - if let Some(e) = error.downcast_ref::() { - return Self::Python(e.clone().into()); + if let Some(e) = error.downcast_ref::() { + return e.cause.clone(); } if let Some(e) = error.source() { diff --git a/crates/sail-common-datafusion/src/extension.rs b/crates/sail-common-datafusion/src/extension.rs index c42df9053..30fbbd03a 100644 --- a/crates/sail-common-datafusion/src/extension.rs +++ b/crates/sail-common-datafusion/src/extension.rs @@ -4,7 +4,7 @@ use datafusion::catalog::Session; use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::execution::{SessionState, TaskContext}; use datafusion::prelude::SessionContext; -use datafusion_common::{internal_datafusion_err, Result}; +use datafusion_common::{Result, internal_datafusion_err}; pub trait SessionExtension: Send + Sync + 'static { fn name() -> &'static str; diff --git a/crates/sail-common-datafusion/src/formatter.rs b/crates/sail-common-datafusion/src/formatter.rs index 43eba8931..9c80c3c82 100644 --- a/crates/sail-common-datafusion/src/formatter.rs +++ b/crates/sail-common-datafusion/src/formatter.rs @@ -121,7 +121,12 @@ pub struct TimestampMicrosecondFormatter<'a>(pub i64, pub Option<&'a Tz>); impl Display for TimestampMicrosecondFormatter<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let Self(microseconds, tz) = self; - match Utc.timestamp_micros(*microseconds).earliest() { + // Convert microseconds to seconds and nanoseconds + // Use div_euclid/rem_euclid to handle negative values correctly + let secs = microseconds.div_euclid(1_000_000); + let nanos = microseconds.rem_euclid(1_000_000) * 1_000; // Convert remaining microseconds to nanoseconds + let datetime = Utc.timestamp_opt(secs, nanos as u32).earliest(); + match datetime { Some(datetime) => write_timestamp(f, &datetime, *tz), None => write!(f, "ERROR"), } @@ -263,7 +268,7 @@ impl Display for IntervalDayTimeFormatter { pub struct IntervalMonthDayNanoFormatter(pub IntervalMonthDayNano); macro_rules! write_interval_part { - ($f:expr, $sep:expr, $value:expr, $unit:expr) => { + ($f:expr_2021, $sep:expr_2021, $value:expr_2021, $unit:expr_2021) => { if $value != 0 { write!($f, "{}{} {}", $sep, $value, $unit)?; " " diff --git a/crates/sail-common-datafusion/src/literal.rs b/crates/sail-common-datafusion/src/literal.rs index 8fd451239..8f71d1347 100644 --- a/crates/sail-common-datafusion/src/literal.rs +++ b/crates/sail-common-datafusion/src/literal.rs @@ -1,6 +1,6 @@ use datafusion::arrow::array::{RecordBatch, RecordBatchOptions}; use datafusion::physical_expr::create_physical_expr; -use datafusion_common::{exec_datafusion_err, exec_err, DFSchema, Result, ScalarValue}; +use datafusion_common::{DFSchema, Result, ScalarValue, exec_datafusion_err, exec_err}; use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::{ColumnarValue, Expr}; diff --git a/crates/sail-common-datafusion/src/logical_rewriter.rs b/crates/sail-common-datafusion/src/logical_rewriter.rs index 735ea3607..1f682e81e 100644 --- a/crates/sail-common-datafusion/src/logical_rewriter.rs +++ b/crates/sail-common-datafusion/src/logical_rewriter.rs @@ -1,5 +1,5 @@ -use datafusion_common::tree_node::Transformed; use datafusion_common::Result; +use datafusion_common::tree_node::Transformed; use datafusion_expr::LogicalPlan; /// A trait for rewriting logical plans after logical optimization. diff --git a/crates/sail-common-datafusion/src/rename/physical_plan.rs b/crates/sail-common-datafusion/src/rename/physical_plan.rs index 51ef2e5c3..07354a0df 100644 --- a/crates/sail-common-datafusion/src/rename/physical_plan.rs +++ b/crates/sail-common-datafusion/src/rename/physical_plan.rs @@ -1,9 +1,9 @@ use std::sync::Arc; -use datafusion::physical_expr::expressions::Column; use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_expr::expressions::Column; use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::projection::ProjectionExec; use datafusion_common::exec_err; pub fn rename_physical_plan( diff --git a/crates/sail-common-datafusion/src/rename/table_provider.rs b/crates/sail-common-datafusion/src/rename/table_provider.rs index 8ed3177e8..b756e19f4 100644 --- a/crates/sail-common-datafusion/src/rename/table_provider.rs +++ b/crates/sail-common-datafusion/src/rename/table_provider.rs @@ -7,7 +7,7 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::Session; use datafusion::datasource::TableProvider; use datafusion::physical_plan::ExecutionPlan; -use datafusion_common::{internal_err, Constraints, Result, Statistics}; +use datafusion_common::{Constraints, Result, Statistics, internal_err}; use datafusion_expr::dml::InsertOp; use datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableType}; diff --git a/crates/sail-common-datafusion/src/schema_evolution.rs b/crates/sail-common-datafusion/src/schema_evolution.rs index a0d88fc9c..d1510bdd6 100644 --- a/crates/sail-common-datafusion/src/schema_evolution.rs +++ b/crates/sail-common-datafusion/src/schema_evolution.rs @@ -1,24 +1,41 @@ use std::sync::Arc; use datafusion::arrow::array::{ - new_null_array, Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, MapArray, - StructArray, + Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, MapArray, StructArray, + new_null_array, }; -use datafusion::arrow::compute::{can_cast_types, cast_with_options, CastOptions}; +use datafusion::arrow::compute::{CastOptions, can_cast_types, cast_with_options}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; -use datafusion::common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion::common::{DataFusionError, Result, ScalarValue, exec_err}; use datafusion::functions::core::getfield::GetFieldFunc; +use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; use datafusion::physical_expr::expressions::{self, Column, Literal}; use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; use datafusion::physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapterFactory}; use datafusion::physical_plan::ColumnarValue; use datafusion_common::format::DEFAULT_CAST_OPTIONS; -use parquet_variant_compute::{unshred_variant, VariantArray}; +use parquet_variant_compute::{VariantArray, unshred_variant}; +use crate::array::record_batch::cast_array_recursively; use crate::variant::{is_binary_variant_field, is_variant_arrow_field, is_variant_storage_type}; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub enum StructFieldMatching { + #[default] + Name, + PhysicalName, + FieldId, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub enum SchemaEvolutionTimezoneMode { + #[default] + Strict, + Relaxed, +} + #[derive(Debug)] pub struct SchemaEvolutionPhysicalExprAdapterFactory {} @@ -28,18 +45,71 @@ impl PhysicalExprAdapterFactory for SchemaEvolutionPhysicalExprAdapterFactory { logical_file_schema: SchemaRef, physical_file_schema: SchemaRef, ) -> Result> { - let (column_mapping, default_values) = - create_column_mapping(&logical_file_schema, &physical_file_schema); + create_schema_evolution_adapter( + logical_file_schema, + physical_file_schema, + StructFieldMatching::Name, + SchemaEvolutionTimezoneMode::Strict, + ) + } +} - Ok(Arc::new(SchemaEvolutionPhysicalExprAdapter { +#[derive(Debug)] +pub struct SchemaEvolutionPhysicalExprAdapterFactoryWithMatching { + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, +} + +impl SchemaEvolutionPhysicalExprAdapterFactoryWithMatching { + pub fn new(matching: StructFieldMatching) -> Self { + Self { + matching, + timezone_mode: SchemaEvolutionTimezoneMode::Strict, + } + } + + pub fn new_relaxed_timezone(matching: StructFieldMatching) -> Self { + Self { + matching, + timezone_mode: SchemaEvolutionTimezoneMode::Relaxed, + } + } +} + +impl PhysicalExprAdapterFactory for SchemaEvolutionPhysicalExprAdapterFactoryWithMatching { + fn create( + &self, + logical_file_schema: SchemaRef, + physical_file_schema: SchemaRef, + ) -> Result> { + create_schema_evolution_adapter( logical_file_schema, physical_file_schema, - column_mapping, - default_values, - })) + self.matching, + self.timezone_mode, + ) } } +fn create_schema_evolution_adapter( + logical_file_schema: SchemaRef, + physical_file_schema: SchemaRef, + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, +) -> Result> { + let (column_mapping, default_values) = + create_column_mapping(&logical_file_schema, &physical_file_schema); + + Ok(Arc::new(SchemaEvolutionPhysicalExprAdapter { + logical_file_schema, + physical_file_schema, + column_mapping, + default_values, + matching, + timezone_mode, + })) +} + fn create_column_mapping( logical_schema: &Schema, physical_schema: &Schema, @@ -80,6 +150,8 @@ struct SchemaEvolutionPhysicalExprAdapter { physical_file_schema: SchemaRef, column_mapping: Vec>, default_values: Vec>, + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, } impl PhysicalExprAdapter for SchemaEvolutionPhysicalExprAdapter { @@ -89,6 +161,8 @@ impl PhysicalExprAdapter for SchemaEvolutionPhysicalExprAdapter { physical_file_schema: &self.physical_file_schema, column_mapping: &self.column_mapping, default_values: &self.default_values, + matching: self.matching, + timezone_mode: self.timezone_mode, }; expr.transform(|expr| rewriter.rewrite_expr(Arc::clone(&expr))) .data() @@ -100,6 +174,8 @@ struct SchemaEvolutionPhysicalExprRewriter<'a> { physical_file_schema: &'a Schema, column_mapping: &'a [Option], default_values: &'a [Option], + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, } impl<'a> SchemaEvolutionPhysicalExprRewriter<'a> { @@ -157,13 +233,6 @@ impl<'a> SchemaEvolutionPhysicalExprRewriter<'a> { DataType::Struct(fields) => fields, _ => return Ok(None), }; - if physical_struct_fields - .iter() - .any(|f| f.name() == field_name) - { - return Ok(None); - } - let logical_field = match self.logical_file_schema.field_with_name(column.name()) { Ok(field) => field, Err(_) => return Ok(None), @@ -179,6 +248,12 @@ impl<'a> SchemaEvolutionPhysicalExprRewriter<'a> { Some(field) => field, None => return Ok(None), }; + if find_matching_struct_field(physical_struct_fields, logical_struct_field, self.matching) + .is_some() + { + return Ok(None); + } + let null_value = ScalarValue::Null.cast_to(logical_struct_field.data_type())?; Ok(Some(Arc::new(Literal::new(null_value)))) } @@ -228,7 +303,10 @@ impl<'a> SchemaEvolutionPhysicalExprRewriter<'a> { let null_value = ScalarValue::Null.cast_to(logical_field.data_type())?; Ok(Transformed::yes(Arc::new(Literal::new(null_value)))) } else { - exec_err!("Non-nullable column '{}' is missing from physical schema and no default value provided", column.name()) + exec_err!( + "Non-nullable column '{}' is missing from physical schema and no default value provided", + column.name() + ) } } None => exec_err!( @@ -271,7 +349,7 @@ impl<'a> SchemaEvolutionPhysicalExprRewriter<'a> { logical_field: &Field, physical_field: &Field, ) -> Result>> { - if !can_cast_field_with_schema_evolution(physical_field, logical_field)? { + if !can_cast_field_with_schema_evolution(physical_field, logical_field, self.matching)? { return exec_err!( "Cannot cast column '{}' from '{}' (physical) to '{}' (logical)", logical_field.name(), @@ -280,18 +358,37 @@ impl<'a> SchemaEvolutionPhysicalExprRewriter<'a> { ); } - Ok(Transformed::yes(Arc::new( - SchemaEvolutionCastColumnExpr::new( - column_expr, - Arc::new(physical_field.clone()), - Arc::new(logical_field.clone()), - None, - ), - ))) + let input_field = Arc::new(physical_field.clone()); + let target_field = Arc::new(logical_field.clone()); + let cast = match self.timezone_mode { + SchemaEvolutionTimezoneMode::Strict => { + SchemaEvolutionCastColumnExpr::new_with_matching( + column_expr, + input_field, + target_field, + None, + self.matching, + ) + } + SchemaEvolutionTimezoneMode::Relaxed => { + SchemaEvolutionCastColumnExpr::new_relaxed_timezone( + column_expr, + input_field, + target_field, + None, + self.matching, + ) + } + }; + Ok(Transformed::yes(Arc::new(cast))) } } -fn can_cast_field_with_schema_evolution(source: &Field, target: &Field) -> Result { +fn can_cast_field_with_schema_evolution( + source: &Field, + target: &Field, + matching: StructFieldMatching, +) -> Result { if source.data_type() == &DataType::Null { return Ok(target.is_nullable()); } @@ -310,14 +407,14 @@ fn can_cast_field_with_schema_evolution(source: &Field, target: &Field) -> Resul match (source.data_type(), target.data_type()) { (DataType::Struct(from_fields), DataType::Struct(to_fields)) => { - validate_struct_compatibility_with_variant(from_fields, to_fields)?; + validate_struct_compatibility_with_variant(from_fields, to_fields, matching)?; Ok(true) } (DataType::List(from_elem), DataType::List(to_elem)) => { - can_cast_field_with_schema_evolution(from_elem, to_elem) + can_cast_field_with_schema_evolution(from_elem, to_elem, matching) } (DataType::LargeList(from_elem), DataType::LargeList(to_elem)) => { - can_cast_field_with_schema_evolution(from_elem, to_elem) + can_cast_field_with_schema_evolution(from_elem, to_elem, matching) } ( DataType::FixedSizeList(from_elem, from_len), @@ -326,24 +423,72 @@ fn can_cast_field_with_schema_evolution(source: &Field, target: &Field) -> Resul if from_len != to_len { return Ok(false); } - can_cast_field_with_schema_evolution(from_elem, to_elem) + can_cast_field_with_schema_evolution(from_elem, to_elem, matching) } (DataType::Map(from_entries, _), DataType::Map(to_entries, _)) => { - can_cast_field_with_schema_evolution(from_entries, to_entries) + validate_map_entries_compatibility(from_entries, to_entries, matching)?; + Ok(true) } _ => Ok(can_cast_types(source.data_type(), target.data_type())), } } +const DELTA_COLUMN_MAPPING_PHYSICAL_NAME_METADATA_KEY: &str = "delta.columnMapping.physicalName"; +// Delta kernel schemas, Arrow Parquet readers, and Delta's physical schema builder use +// different field-ID metadata spellings, all of which represent the same identity. +const STRUCT_FIELD_ID_METADATA_KEYS: [&str; 3] = [ + "delta.columnMapping.id", + PARQUET_FIELD_ID_META_KEY, + "parquet.field.id", +]; + +fn struct_field_id(field: &Field) -> Option<&str> { + STRUCT_FIELD_ID_METADATA_KEYS + .iter() + .find_map(|key| field.metadata().get(*key).map(String::as_str)) +} + +fn find_matching_struct_field<'a>( + source_fields: &'a [Arc], + target_field: &Field, + matching: StructFieldMatching, +) -> Option<(usize, &'a Arc)> { + match matching { + StructFieldMatching::Name => source_fields + .iter() + .enumerate() + .find(|(_, source)| source.name() == target_field.name()), + StructFieldMatching::PhysicalName => { + // Synthetic list/map wrappers are matched by their container handlers; only + // actual Delta struct fields with physicalName metadata reach this branch. + let physical_name = target_field + .metadata() + .get(DELTA_COLUMN_MAPPING_PHYSICAL_NAME_METADATA_KEY)?; + source_fields + .iter() + .enumerate() + .find(|(_, source)| source.name() == physical_name) + } + StructFieldMatching::FieldId => { + let target_id = struct_field_id(target_field)?; + source_fields + .iter() + .enumerate() + .find(|(_, source)| struct_field_id(source) == Some(target_id)) + } + } +} + fn validate_struct_compatibility_with_variant( source_fields: &[Arc], target_fields: &[Arc], + matching: StructFieldMatching, ) -> Result<()> { - if !target_fields.iter().any(|target| { - source_fields + if matching == StructFieldMatching::Name + && !target_fields .iter() - .any(|source| source.name() == target.name()) - }) { + .any(|target| find_matching_struct_field(source_fields, target, matching).is_some()) + { return exec_err!( "Cannot cast struct with {} fields to {} fields because there is no field name overlap", source_fields.len(), @@ -352,12 +497,9 @@ fn validate_struct_compatibility_with_variant( } for target_field in target_fields { - match source_fields - .iter() - .find(|source| source.name() == target_field.name()) - { - Some(source_field) => { - if !can_cast_field_with_schema_evolution(source_field, target_field)? { + match find_matching_struct_field(source_fields, target_field, matching) { + Some((_, source_field)) => { + if !can_cast_field_with_schema_evolution(source_field, target_field, matching)? { return exec_err!( "Cannot cast struct field '{}' from type {} to type {}", target_field.name(), @@ -380,12 +522,55 @@ fn validate_struct_compatibility_with_variant( Ok(()) } +fn validate_map_entries_compatibility( + source_entries: &Field, + target_entries: &Field, + matching: StructFieldMatching, +) -> Result<()> { + let (DataType::Struct(source_fields), DataType::Struct(target_fields)) = + (source_entries.data_type(), target_entries.data_type()) + else { + return exec_err!("map entries must use struct types"); + }; + + for target_field in target_fields { + // Arrow map key/value fields are structural wrappers without column-mapping metadata. + match source_fields + .iter() + .find(|source| source.name() == target_field.name()) + { + Some(source_field) + if can_cast_field_with_schema_evolution(source_field, target_field, matching)? => {} + Some(source_field) => { + return exec_err!( + "Cannot cast map entry '{}' from type {} to type {}", + target_field.name(), + source_field.data_type(), + target_field.data_type() + ); + } + None if target_field.is_nullable() => {} + None => { + return exec_err!( + "Cannot cast map: target entry '{}' is non-nullable but missing from source. \ + Cannot fill with NULL.", + target_field.name() + ); + } + } + } + + Ok(()) +} + #[derive(Debug, Clone, Eq)] pub struct SchemaEvolutionCastColumnExpr { expr: Arc, input_field: Arc, target_field: Arc, cast_options: CastOptions<'static>, + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, } impl PartialEq for SchemaEvolutionCastColumnExpr { @@ -394,6 +579,8 @@ impl PartialEq for SchemaEvolutionCastColumnExpr { && self.input_field.eq(&other.input_field) && self.target_field.eq(&other.target_field) && self.cast_options.eq(&other.cast_options) + && self.matching.eq(&other.matching) + && self.timezone_mode.eq(&other.timezone_mode) } } @@ -403,6 +590,8 @@ impl std::hash::Hash for SchemaEvolutionCastColumnExpr { self.input_field.hash(state); self.target_field.hash(state); self.cast_options.hash(state); + self.matching.hash(state); + self.timezone_mode.hash(state); } } @@ -423,12 +612,65 @@ impl SchemaEvolutionCastColumnExpr { input_field: Arc, target_field: Arc, cast_options: Option>, + ) -> Self { + Self::new_with_matching( + expr, + input_field, + target_field, + cast_options, + StructFieldMatching::Name, + ) + } + + pub fn new_with_matching( + expr: Arc, + input_field: Arc, + target_field: Arc, + cast_options: Option>, + matching: StructFieldMatching, + ) -> Self { + Self::from_parts( + expr, + input_field, + target_field, + cast_options, + matching, + SchemaEvolutionTimezoneMode::Strict, + ) + } + + pub fn new_relaxed_timezone( + expr: Arc, + input_field: Arc, + target_field: Arc, + cast_options: Option>, + matching: StructFieldMatching, + ) -> Self { + Self::from_parts( + expr, + input_field, + target_field, + cast_options, + matching, + SchemaEvolutionTimezoneMode::Relaxed, + ) + } + + fn from_parts( + expr: Arc, + input_field: Arc, + target_field: Arc, + cast_options: Option>, + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, ) -> Self { Self { expr, input_field, target_field, cast_options: cast_options.unwrap_or(DEFAULT_CAST_OPTIONS), + matching, + timezone_mode, } } @@ -439,6 +681,31 @@ impl SchemaEvolutionCastColumnExpr { pub fn target_field(&self) -> &Arc { &self.target_field } + + pub const fn matching(&self) -> StructFieldMatching { + self.matching + } + + pub const fn timezone_mode(&self) -> SchemaEvolutionTimezoneMode { + self.timezone_mode + } + + fn cast_array(&self, array: &ArrayRef) -> Result { + match self.timezone_mode { + SchemaEvolutionTimezoneMode::Strict => cast_array_with_schema_evolution( + array, + self.target_field.as_ref(), + &self.cast_options, + self.matching, + ), + SchemaEvolutionTimezoneMode::Relaxed => cast_array_with_schema_evolution_relaxed_tz( + array, + self.target_field.as_ref(), + &self.cast_options, + self.matching, + ), + } + } } impl PhysicalExpr for SchemaEvolutionCastColumnExpr { @@ -453,20 +720,15 @@ impl PhysicalExpr for SchemaEvolutionCastColumnExpr { fn evaluate(&self, batch: &RecordBatch) -> Result { let value = self.expr.evaluate(batch)?; match value { - ColumnarValue::Array(array) => { - Ok(ColumnarValue::Array(cast_array_with_schema_evolution( - &array, - self.target_field.as_ref(), - &self.cast_options, - )?)) + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(self.cast_array(&array)?)), + ColumnarValue::Scalar(scalar) + if scalar.data_type() == *self.target_field.data_type() => + { + Ok(ColumnarValue::Scalar(scalar)) } ColumnarValue::Scalar(scalar) => { let as_array = scalar.to_array_of_size(1)?; - let casted = cast_array_with_schema_evolution( - &as_array, - self.target_field.as_ref(), - &self.cast_options, - )?; + let casted = self.cast_array(&as_array)?; Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( casted.as_ref(), 0, @@ -491,11 +753,13 @@ impl PhysicalExpr for SchemaEvolutionCastColumnExpr { let child = children.pop().ok_or_else(|| { DataFusionError::Plan("SchemaEvolutionCastColumnExpr requires a child".to_string()) })?; - Ok(Arc::new(Self::new( + Ok(Arc::new(Self::from_parts( child, Arc::clone(&self.input_field), Arc::clone(&self.target_field), Some(self.cast_options.clone()), + self.matching, + self.timezone_mode, ))) } @@ -504,19 +768,58 @@ impl PhysicalExpr for SchemaEvolutionCastColumnExpr { } } -fn cast_array_with_schema_evolution( +pub fn cast_array_with_schema_evolution( + source: &ArrayRef, + target_field: &Field, + cast_options: &CastOptions, + matching: StructFieldMatching, +) -> Result { + cast_array_with_schema_evolution_inner(source, target_field, cast_options, matching, false) +} + +pub fn cast_array_with_schema_evolution_relaxed_tz( + source: &ArrayRef, + target_field: &Field, + cast_options: &CastOptions, + matching: StructFieldMatching, +) -> Result { + cast_array_with_schema_evolution_inner(source, target_field, cast_options, matching, true) +} + +fn cast_array_with_schema_evolution_inner( source: &ArrayRef, target_field: &Field, cast_options: &CastOptions, + matching: StructFieldMatching, + relaxed_timezone: bool, ) -> Result { if is_variant_arrow_field(target_field) && is_variant_storage_type(source.data_type()) { return cast_variant_array_with_schema_evolution(source, target_field, cast_options); } + if source.data_type() == target_field.data_type() { + return Ok(Arc::clone(source)); + } + + if relaxed_timezone + && let (DataType::Timestamp(source_unit, _), DataType::Timestamp(target_unit, _)) = + (source.data_type(), target_field.data_type()) + { + let source_without_timezone = DataType::Timestamp(*source_unit, None); + let source = cast_array_recursively(source, &source_without_timezone)?; + let target_without_timezone = DataType::Timestamp(*target_unit, None); + let scaled = cast_with_options(&source, &target_without_timezone, cast_options)?; + return cast_array_recursively(&scaled, target_field.data_type()); + } + match target_field.data_type() { - DataType::Struct(target_fields) => { - cast_struct_array_with_schema_evolution(source, target_fields, cast_options) - } + DataType::Struct(target_fields) => cast_struct_array_with_schema_evolution( + source, + target_fields, + cast_options, + matching, + relaxed_timezone, + ), DataType::List(target_elem) => { let Some(source_list) = source.as_any().downcast_ref::() else { return exec_err!( @@ -524,10 +827,12 @@ fn cast_array_with_schema_evolution( source.data_type() ); }; - let casted_values = cast_array_with_schema_evolution( + let casted_values = cast_array_with_schema_evolution_inner( source_list.values(), target_elem.as_ref(), cast_options, + matching, + relaxed_timezone, )?; Ok(Arc::new(ListArray::new( Arc::clone(target_elem), @@ -543,10 +848,12 @@ fn cast_array_with_schema_evolution( source.data_type() ); }; - let casted_values = cast_array_with_schema_evolution( + let casted_values = cast_array_with_schema_evolution_inner( source_list.values(), target_elem.as_ref(), cast_options, + matching, + relaxed_timezone, )?; Ok(Arc::new(LargeListArray::new( Arc::clone(target_elem), @@ -570,10 +877,12 @@ fn cast_array_with_schema_evolution( target_len ); } - let casted_values = cast_array_with_schema_evolution( + let casted_values = cast_array_with_schema_evolution_inner( source_list.values(), target_elem.as_ref(), cast_options, + matching, + relaxed_timezone, )?; Ok(Arc::new(FixedSizeListArray::new( Arc::clone(target_elem), @@ -602,12 +911,15 @@ fn cast_array_with_schema_evolution( let mut kv_fields: Vec> = Vec::with_capacity(target_kv_fields.len()); for target_child in target_kv_fields { + // Match the synthetic key/value wrapper by name, then apply mapping recursively. kv_fields.push(Arc::clone(target_child)); match source_map.entries().column_by_name(target_child.name()) { - Some(source_child) => kv_arrays.push(cast_array_with_schema_evolution( + Some(source_child) => kv_arrays.push(cast_array_with_schema_evolution_inner( source_child, target_child.as_ref(), cast_options, + matching, + relaxed_timezone, )?), None => kv_arrays.push(new_null_array(target_child.data_type(), num_entries)), } @@ -644,13 +956,21 @@ fn cast_variant_array_with_schema_evolution( let variant = VariantArray::try_new(source.as_ref())?; let unshredded = unshred_variant(&variant)?; let unshredded = Arc::new(unshredded.into_inner()) as ArrayRef; - cast_struct_array_to_fields(&unshredded, target_fields, cast_options) + cast_struct_array_to_fields( + &unshredded, + target_fields, + cast_options, + StructFieldMatching::Name, + false, + ) } fn cast_struct_array_with_schema_evolution( source: &ArrayRef, target_fields: &[Arc], cast_options: &CastOptions, + matching: StructFieldMatching, + relaxed_timezone: bool, ) -> Result { if source.data_type() == &DataType::Null || (!source.is_empty() && source.null_count() == source.len()) @@ -667,14 +987,22 @@ fn cast_struct_array_with_schema_evolution( source.data_type() ); }; - validate_struct_compatibility_with_variant(source_struct.fields(), target_fields)?; - cast_struct_array_to_fields(source, target_fields, cast_options) + validate_struct_compatibility_with_variant(source_struct.fields(), target_fields, matching)?; + cast_struct_array_to_fields( + source, + target_fields, + cast_options, + matching, + relaxed_timezone, + ) } fn cast_struct_array_to_fields( source: &ArrayRef, target_fields: &[Arc], cast_options: &CastOptions, + matching: StructFieldMatching, + relaxed_timezone: bool, ) -> Result { let Some(source_struct) = source.as_any().downcast_ref::() else { return exec_err!( @@ -688,12 +1016,14 @@ fn cast_struct_array_to_fields( for target_child in target_fields { fields.push(Arc::clone(target_child)); - match source_struct.column_by_name(target_child.name()) { - Some(source_child) => { - arrays.push(cast_array_with_schema_evolution( - source_child, + match find_matching_struct_field(source_struct.fields(), target_child, matching) { + Some((source_index, _)) => { + arrays.push(cast_array_with_schema_evolution_inner( + source_struct.column(source_index), target_child.as_ref(), cast_options, + matching, + relaxed_timezone, )?); } None => arrays.push(new_null_array(target_child.data_type(), num_rows)), @@ -710,9 +1040,15 @@ fn cast_struct_array_to_fields( #[cfg(test)] #[expect(clippy::expect_used, clippy::unwrap_used)] mod tests { - use datafusion::arrow::array::{BinaryViewArray, Int64Array, StringArray}; + use std::collections::HashMap; + + use datafusion::arrow::array::{ + BinaryViewArray, Int64Array, StringArray, TimestampMicrosecondArray, + TimestampMillisecondArray, + }; use datafusion::arrow::buffer::OffsetBuffer; - use parquet_variant_compute::{json_to_variant, shred_variant, variant_to_json, VariantType}; + use datafusion::arrow::datatypes::TimeUnit; + use parquet_variant_compute::{VariantType, json_to_variant, shred_variant, variant_to_json}; use super::*; @@ -731,6 +1067,395 @@ mod tests { .with_extension_type(VariantType) } + fn field_with_id(name: &str, metadata_key: &str, id: i64) -> Arc { + Arc::new( + Field::new(name, DataType::Int64, true) + .with_metadata(HashMap::from([(metadata_key.to_string(), id.to_string())])), + ) + } + + #[test] + fn cast_struct_fields_by_column_id() -> Result<()> { + let source_fields = vec![ + field_with_id("col-a", PARQUET_FIELD_ID_META_KEY, 1), + field_with_id("col-b", PARQUET_FIELD_ID_META_KEY, 2), + ]; + let source = Arc::new(StructArray::new( + source_fields.into(), + vec![ + Arc::new(Int64Array::from(vec![Some(10)])), + Arc::new(Int64Array::from(vec![Some(20)])), + ], + None, + )) as ArrayRef; + let target_fields = vec![ + field_with_id("logical_b", "delta.columnMapping.id", 2), + field_with_id("logical_a", "delta.columnMapping.id", 1), + ]; + let target_field = Field::new("root", DataType::Struct(target_fields.clone().into()), true); + + assert!(can_cast_field_with_schema_evolution( + &Field::new("root", source.data_type().clone(), true), + &target_field, + StructFieldMatching::FieldId, + )?); + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::FieldId, + )?; + let casted = casted + .as_any() + .downcast_ref::() + .expect("struct array"); + + assert!(casted.fields().iter().eq(target_fields.iter())); + assert_eq!( + casted + .column_by_name("logical_b") + .expect("logical_b") + .as_any() + .downcast_ref::() + .expect("int64") + .value(0), + 20, + ); + assert_eq!( + casted + .column_by_name("logical_a") + .expect("logical_a") + .as_any() + .downcast_ref::() + .expect("int64") + .value(0), + 10, + ); + Ok(()) + } + + #[test] + fn generic_default_matches_nested_fields_by_name() -> Result<()> { + let source_by_name = field_with_id("logical", PARQUET_FIELD_ID_META_KEY, 2); + let source_by_id = field_with_id("old_name", PARQUET_FIELD_ID_META_KEY, 1); + let source = Arc::new(StructArray::new( + vec![source_by_name, source_by_id].into(), + vec![ + Arc::new(Int64Array::from(vec![Some(20)])), + Arc::new(Int64Array::from(vec![Some(10)])), + ], + None, + )) as ArrayRef; + let target_child = field_with_id("logical", "delta.columnMapping.id", 1); + let target_field = Field::new("root", DataType::Struct(vec![target_child].into()), true); + + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::default(), + )?; + let casted = casted + .as_any() + .downcast_ref::() + .expect("struct array"); + assert_eq!( + casted + .column_by_name("logical") + .expect("logical") + .as_any() + .downcast_ref::() + .expect("int64") + .value(0), + 20, + ); + Ok(()) + } + + #[test] + fn generic_name_mode_rejects_structs_without_field_overlap() { + let source = Arc::new(StructArray::new( + vec![Arc::new(Field::new("old", DataType::Int64, true))].into(), + vec![Arc::new(Int64Array::from(vec![Some(10)]))], + None, + )) as ArrayRef; + let target_field = Field::new( + "root", + DataType::Struct(vec![Arc::new(Field::new("new", DataType::Int64, true))].into()), + true, + ); + + let error = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::Name, + ) + .expect_err("zero-overlap name-based structs must fail"); + + assert!(error.to_string().contains("no field name overlap")); + } + + #[test] + fn relaxed_timezone_cast_scales_without_shifting_values() -> Result<()> { + let source = Arc::new(TimestampMillisecondArray::from(vec![Some(1_000)])) as ArrayRef; + let target = Field::new( + "ts", + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("Europe/Amsterdam"))), + true, + ); + + let casted = cast_array_with_schema_evolution_relaxed_tz( + &source, + &target, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::Name, + )?; + let timestamp = casted + .as_any() + .downcast_ref::() + .expect("timestamp"); + + assert_eq!(timestamp.value(0), 1_000_000); + assert_eq!(timestamp.data_type(), target.data_type()); + Ok(()) + } + + #[test] + fn identical_schema_evolution_reuses_array() -> Result<()> { + let source = Arc::new(Int64Array::from(vec![Some(10), None])) as ArrayRef; + let target = Field::new("value", DataType::Int64, true); + + let casted = cast_array_with_schema_evolution_relaxed_tz( + &source, + &target, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::Name, + )?; + + assert!(Arc::ptr_eq(&source, &casted)); + Ok(()) + } + + #[test] + fn relaxed_timezone_cast_recurses_into_structs() -> Result<()> { + let source_child = Arc::new(Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::from("UTC"))), + true, + )); + let source = Arc::new(StructArray::new( + vec![Arc::clone(&source_child)].into(), + vec![Arc::new( + TimestampMillisecondArray::from(vec![Some(1_000)]).with_timezone("UTC"), + )], + None, + )) as ArrayRef; + let target_child = Arc::new(Field::new( + "ts", + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("Europe/Amsterdam"))), + true, + )); + let target = Field::new( + "root", + DataType::Struct(vec![Arc::clone(&target_child)].into()), + true, + ); + + let casted = cast_array_with_schema_evolution_relaxed_tz( + &source, + &target, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::Name, + )?; + let casted = casted + .as_any() + .downcast_ref::() + .expect("struct"); + let timestamp = casted + .column(0) + .as_any() + .downcast_ref::() + .expect("timestamp"); + + assert_eq!(timestamp.value(0), 1_000_000); + assert_eq!(timestamp.data_type(), target_child.data_type()); + Ok(()) + } + + #[test] + fn name_mode_does_not_fallback_to_id_or_logical_name() -> Result<()> { + let source_child = field_with_id("logical", PARQUET_FIELD_ID_META_KEY, 1); + let source = Arc::new(StructArray::new( + vec![source_child].into(), + vec![Arc::new(Int64Array::from(vec![Some(10)]))], + None, + )) as ArrayRef; + let target_child = Arc::new(Field::new("logical", DataType::Int64, true).with_metadata( + HashMap::from([ + ( + DELTA_COLUMN_MAPPING_PHYSICAL_NAME_METADATA_KEY.to_string(), + "different-physical-name".to_string(), + ), + ("delta.columnMapping.id".to_string(), "1".to_string()), + ]), + )); + let target_field = Field::new("root", DataType::Struct(vec![target_child].into()), true); + + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::PhysicalName, + )?; + let casted = casted + .as_any() + .downcast_ref::() + .expect("struct array"); + + assert!(casted.column(0).is_null(0)); + Ok(()) + } + + #[test] + fn missing_non_nullable_mapped_field_is_rejected() { + let source = Arc::new(StructArray::new( + vec![Arc::new(Field::new("other", DataType::Int64, true))].into(), + vec![Arc::new(Int64Array::from(vec![Some(10)]))], + None, + )) as ArrayRef; + let target_child = Arc::new( + Field::new("required", DataType::Int64, false).with_metadata(HashMap::from([( + DELTA_COLUMN_MAPPING_PHYSICAL_NAME_METADATA_KEY.to_string(), + "required-physical".to_string(), + )])), + ); + let target_field = Field::new("root", DataType::Struct(vec![target_child].into()), true); + + let error = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::PhysicalName, + ) + .expect_err("missing non-nullable field must fail"); + + assert!( + error + .to_string() + .contains("target field 'required' is non-nullable") + ); + } + + #[test] + fn map_entry_wrappers_match_by_name_in_physical_name_mode() -> Result<()> { + let source_value = Arc::new(Field::new( + "value", + DataType::Struct(vec![Arc::new(Field::new("col-a", DataType::Int64, true))].into()), + true, + )); + let source = Field::new( + "attrs", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Arc::new(Field::new("key", DataType::Utf8, false)), + source_value, + ] + .into(), + ), + false, + )), + false, + ), + true, + ); + + let target_value_child = Arc::new(Field::new("a", DataType::Int64, true).with_metadata( + HashMap::from([( + DELTA_COLUMN_MAPPING_PHYSICAL_NAME_METADATA_KEY.to_string(), + "col-a".to_string(), + )]), + )); + let target = Field::new( + "attrs", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Arc::new(Field::new("key", DataType::Utf8, false)), + Arc::new(Field::new( + "value", + DataType::Struct(vec![target_value_child].into()), + true, + )), + ] + .into(), + ), + false, + )), + false, + ), + true, + ); + + assert!(can_cast_field_with_schema_evolution( + &source, + &target, + StructFieldMatching::PhysicalName, + )?); + Ok(()) + } + + #[test] + fn cast_struct_field_by_physical_name_before_logical_name() -> Result<()> { + let source = Arc::new(StructArray::new( + vec![ + Arc::new(Field::new("rate", DataType::Int64, true)), + Arc::new(Field::new("col-rate", DataType::Int64, true)), + ] + .into(), + vec![ + Arc::new(Int64Array::from(vec![Some(10)])), + Arc::new(Int64Array::from(vec![Some(20)])), + ], + None, + )) as ArrayRef; + let target_child = Arc::new(Field::new("rate", DataType::Int64, true).with_metadata( + HashMap::from([( + DELTA_COLUMN_MAPPING_PHYSICAL_NAME_METADATA_KEY.to_string(), + "col-rate".to_string(), + )]), + )); + let target_field = Field::new("root", DataType::Struct(vec![target_child].into()), true); + + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::PhysicalName, + )?; + let casted = casted + .as_any() + .downcast_ref::() + .expect("struct array"); + + assert_eq!( + casted + .column_by_name("rate") + .expect("rate") + .as_any() + .downcast_ref::() + .expect("int64") + .value(0), + 20, + ); + Ok(()) + } + #[test] fn cast_array_unshreds_variant_without_value_field() -> Result<()> { let metadata = BinaryViewArray::from_iter_values(vec![&[0x01, 0x00, 0x00][..]]); @@ -749,10 +1474,15 @@ mod tests { assert!(can_cast_field_with_schema_evolution( &source_field, - &target_field + &target_field, + StructFieldMatching::PhysicalName, )?); - let casted = - cast_array_with_schema_evolution(&source, &target_field, &DEFAULT_CAST_OPTIONS)?; + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::PhysicalName, + )?; assert_eq!(casted.data_type(), target_field.data_type()); let json = variant_to_json(&casted)?; @@ -787,8 +1517,12 @@ mod tests { let target_payload = Arc::new(variant_field("payload")); let target_field = Field::new("root", DataType::Struct(vec![target_payload].into()), true); - let casted = - cast_array_with_schema_evolution(&source, &target_field, &DEFAULT_CAST_OPTIONS)?; + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::Name, + )?; let casted_struct = casted .as_any() .downcast_ref::() @@ -825,8 +1559,12 @@ mod tests { true, ); - let casted = - cast_array_with_schema_evolution(&source, &target_field, &DEFAULT_CAST_OPTIONS)?; + let casted = cast_array_with_schema_evolution( + &source, + &target_field, + &DEFAULT_CAST_OPTIONS, + StructFieldMatching::Name, + )?; let casted_list = casted .as_any() .downcast_ref::() diff --git a/crates/sail-common-datafusion/src/session/job.rs b/crates/sail-common-datafusion/src/session/job.rs index 9bf95d292..3fd457115 100644 --- a/crates/sail-common-datafusion/src/session/job.rs +++ b/crates/sail-common-datafusion/src/session/job.rs @@ -5,7 +5,6 @@ use datafusion::common::Result; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionContext; -use tokio::sync::oneshot; use crate::extension::SessionExtension; use crate::system::catalog::{JobRow, StageRow, TaskRow, WorkerRow}; @@ -22,7 +21,15 @@ pub trait JobRunner: StateObservable + Send + Sync + 'static plan: Arc, ) -> Result; - async fn stop(&self, history: oneshot::Sender); + /// Returns the distributed execution plan for EXPLAIN output. + fn explain(&self, plan: Arc) -> Result; + + async fn stop(&self); +} + +#[tonic::async_trait] +pub trait JobRunnerHistoryReporter: Send + 'static { + async fn report(self: Box, history: JobRunnerHistory); } pub struct JobRunnerHistory { diff --git a/crates/sail-common-datafusion/src/session/plan.rs b/crates/sail-common-datafusion/src/session/plan.rs index a2ccc8bf9..1f0dfbf15 100644 --- a/crates/sail-common-datafusion/src/session/plan.rs +++ b/crates/sail-common-datafusion/src/session/plan.rs @@ -2,7 +2,7 @@ use std::fmt::Debug; use datafusion::arrow::datatypes::DataType; use datafusion_common::{Result, ScalarValue}; -use sail_common::object::DynObject; +use sail_common::utils::object::DynObject; use crate::catalog::display::CatalogDisplay; use crate::extension::SessionExtension; diff --git a/crates/sail-common-datafusion/src/streaming/event/encoding.rs b/crates/sail-common-datafusion/src/streaming/event/encoding.rs index 1dcb44bdf..24032110e 100644 --- a/crates/sail-common-datafusion/src/streaming/event/encoding.rs +++ b/crates/sail-common-datafusion/src/streaming/event/encoding.rs @@ -3,19 +3,19 @@ use std::sync::Arc; use std::task::{Context, Poll}; use datafusion::arrow::array::{ - new_null_array, Array, ArrayData, ArrayRef, BinaryArray, BooleanArray, RecordBatch, + Array, ArrayData, ArrayRef, BinaryArray, BooleanArray, RecordBatch, new_null_array, }; use datafusion::arrow::buffer::Buffer; use datafusion::arrow::datatypes::{DataType, SchemaRef}; use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream}; -use datafusion_common::{exec_datafusion_err, exec_err, Result}; +use datafusion_common::{Result, exec_datafusion_err, exec_err}; use futures::{Stream, TryStreamExt}; use crate::array::placeholder::{placeholder_array, placeholder_boolean_array}; +use crate::streaming::event::FlowEvent; use crate::streaming::event::marker::FlowMarker; use crate::streaming::event::schema::{to_flow_event_schema, try_from_flow_event_schema}; use crate::streaming::event::stream::{FlowEventStream, SendableFlowEventStream}; -use crate::streaming::event::FlowEvent; /// A stream for encoded flow events. /// The encoded [`RecordBatch`] has a special schema. diff --git a/crates/sail-common-datafusion/src/streaming/event/marker.rs b/crates/sail-common-datafusion/src/streaming/event/marker.rs index bbcc317b2..7d6f36765 100644 --- a/crates/sail-common-datafusion/src/streaming/event/marker.rs +++ b/crates/sail-common-datafusion/src/streaming/event/marker.rs @@ -1,8 +1,8 @@ use chrono::{DateTime, Utc}; -use datafusion_common::{plan_datafusion_err, plan_err, Result}; +use datafusion_common::{Result, plan_datafusion_err, plan_err}; use prost::Message; -use crate::streaming::event::gen; +use crate::streaming::event::r#gen; /// A marker injected in a streaming data flow for various purposes. #[derive(Debug, Clone, PartialEq, Eq)] @@ -42,33 +42,33 @@ impl FlowMarker { source, id, timestamp, - } => gen::flow_marker::Kind::LatencyTracker(gen::LatencyTracker { + } => r#gen::flow_marker::Kind::LatencyTracker(r#gen::LatencyTracker { source, id, timestamp_secs: timestamp.timestamp(), timestamp_nanos: timestamp.timestamp_subsec_nanos(), }), FlowMarker::Watermark { source, timestamp } => { - gen::flow_marker::Kind::Watermark(gen::Watermark { + r#gen::flow_marker::Kind::Watermark(r#gen::Watermark { source, timestamp_secs: timestamp.timestamp(), timestamp_nanos: timestamp.timestamp_subsec_nanos(), }) } FlowMarker::Checkpoint { id } => { - gen::flow_marker::Kind::Checkpoint(gen::Checkpoint { id }) + r#gen::flow_marker::Kind::Checkpoint(r#gen::Checkpoint { id }) } - FlowMarker::EndOfData => gen::flow_marker::Kind::EndOfData(gen::EndOfData {}), + FlowMarker::EndOfData => r#gen::flow_marker::Kind::EndOfData(r#gen::EndOfData {}), }; - let message = gen::FlowMarker { kind: Some(kind) }; + let message = r#gen::FlowMarker { kind: Some(kind) }; Ok(message.encode_to_vec()) } pub fn decode(bytes: &[u8]) -> Result { - let message = gen::FlowMarker::decode(bytes) + let message = r#gen::FlowMarker::decode(bytes) .map_err(|e| plan_datafusion_err!("failed to decode marker: {e}"))?; match message.kind { - Some(gen::flow_marker::Kind::LatencyTracker(gen::LatencyTracker { + Some(r#gen::flow_marker::Kind::LatencyTracker(r#gen::LatencyTracker { source, id, timestamp_secs, @@ -79,7 +79,7 @@ impl FlowMarker { timestamp: >::from_timestamp(timestamp_secs, timestamp_nanos) .ok_or_else(|| plan_datafusion_err!("invalid latency tracker timestamp"))?, }), - Some(gen::flow_marker::Kind::Watermark(gen::Watermark { + Some(r#gen::flow_marker::Kind::Watermark(r#gen::Watermark { source, timestamp_secs, timestamp_nanos, @@ -88,10 +88,12 @@ impl FlowMarker { timestamp: >::from_timestamp(timestamp_secs, timestamp_nanos) .ok_or_else(|| plan_datafusion_err!("invalid watermark timestamp"))?, }), - Some(gen::flow_marker::Kind::Checkpoint(gen::Checkpoint { id })) => { + Some(r#gen::flow_marker::Kind::Checkpoint(r#gen::Checkpoint { id })) => { Ok(FlowMarker::Checkpoint { id }) } - Some(gen::flow_marker::Kind::EndOfData(gen::EndOfData {})) => Ok(FlowMarker::EndOfData), + Some(r#gen::flow_marker::Kind::EndOfData(r#gen::EndOfData {})) => { + Ok(FlowMarker::EndOfData) + } None => plan_err!("missing marker kind"), } } diff --git a/crates/sail-common-datafusion/src/streaming/event/mod.rs b/crates/sail-common-datafusion/src/streaming/event/mod.rs index 3545103d8..70814eb2e 100644 --- a/crates/sail-common-datafusion/src/streaming/event/mod.rs +++ b/crates/sail-common-datafusion/src/streaming/event/mod.rs @@ -5,7 +5,7 @@ pub mod stream; use datafusion::arrow::array::{BooleanArray, BooleanBuilder, RecordBatch}; -pub(crate) mod gen { +pub(crate) mod r#gen { tonic::include_proto!("sail.streaming"); } diff --git a/crates/sail-common-datafusion/src/streaming/event/schema.rs b/crates/sail-common-datafusion/src/streaming/event/schema.rs index 95649a0b9..9403f2484 100644 --- a/crates/sail-common-datafusion/src/streaming/event/schema.rs +++ b/crates/sail-common-datafusion/src/streaming/event/schema.rs @@ -4,7 +4,7 @@ //! that carry the information about the marker and data retraction. use datafusion::arrow::datatypes::{DataType, Field, Schema}; -use datafusion_common::{plan_err, Result}; +use datafusion_common::{Result, plan_err}; /// The name of the marker field in encoded an [`RecordBatch`] for /// flow events. diff --git a/crates/sail-common-datafusion/src/streaming/source.rs b/crates/sail-common-datafusion/src/streaming/source.rs index 9f915f994..a76872f45 100644 --- a/crates/sail-common-datafusion/src/streaming/source.rs +++ b/crates/sail-common-datafusion/src/streaming/source.rs @@ -6,7 +6,7 @@ use datafusion::catalog::Session; use datafusion::datasource::{TableProvider, TableType}; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; use datafusion::physical_plan::ExecutionPlan; -use datafusion_common::{internal_err, Constraints, Result}; +use datafusion_common::{Constraints, Result, internal_err}; /// A streaming data source. /// The streaming logical plan rewriter will identify such data sources diff --git a/crates/sail-common-datafusion/src/system/predicate.rs b/crates/sail-common-datafusion/src/system/predicate.rs index 101e9c03b..748798b26 100644 --- a/crates/sail-common-datafusion/src/system/predicate.rs +++ b/crates/sail-common-datafusion/src/system/predicate.rs @@ -174,11 +174,14 @@ where return None; } if let Some(current) = &mut self.current { - if let Some(item) = current.next() { - self.fetch -= 1; - return Some(Ok(item)); - } else { - self.current = None; + match current.next() { + Some(item) => { + self.fetch -= 1; + return Some(Ok(item)); + } + _ => { + self.current = None; + } } } for item in self.input.by_ref() { @@ -270,7 +273,7 @@ impl PredicateFilterAsyncFlatMapTask { mod tests { use std::sync::Arc; - use datafusion_common::{internal_err, Result}; + use datafusion_common::{Result, internal_err}; use super::{Predicate, PredicateExt}; diff --git a/crates/sail-common-datafusion/src/system/types.rs b/crates/sail-common-datafusion/src/system/types.rs index fc96df434..507221341 100644 --- a/crates/sail-common-datafusion/src/system/types.rs +++ b/crates/sail-common-datafusion/src/system/types.rs @@ -28,7 +28,7 @@ trait NumericType { } macro_rules! impl_numeric_type { - ($t:ty, $dt:expr) => { + ($t:ty, $dt:expr_2021) => { impl NumericType for $t { fn data_type() -> DataType { $dt diff --git a/crates/sail-common-datafusion/src/udf.rs b/crates/sail-common-datafusion/src/udf.rs index 01b691a03..b0946d272 100644 --- a/crates/sail-common-datafusion/src/udf.rs +++ b/crates/sail-common-datafusion/src/udf.rs @@ -6,7 +6,7 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::execution::SendableRecordBatchStream; use datafusion_common::Result; use sail_common::impl_dyn_object_traits; -use sail_common::object::DynObject; +use sail_common::utils::object::DynObject; /// Trait for implementing UDF to map input stream to output stream. pub trait StreamUDF: DynObject + Debug + Send + Sync { diff --git a/crates/sail-common-datafusion/src/utils/datetime.rs b/crates/sail-common-datafusion/src/utils/datetime.rs index df6b46984..c29d3e09d 100644 --- a/crates/sail-common-datafusion/src/utils/datetime.rs +++ b/crates/sail-common-datafusion/src/utils/datetime.rs @@ -1,5 +1,5 @@ use chrono::{DateTime, Days, NaiveDateTime, TimeZone, Utc}; -use datafusion_common::{exec_datafusion_err, Result}; +use datafusion_common::{Result, exec_datafusion_err}; /// Localize the naive datetime to the time zone. /// 1. If the datatime is mapped to exactly one datetime in the time zone, the local datetime diff --git a/crates/sail-common-datafusion/src/utils/items.rs b/crates/sail-common-datafusion/src/utils/items.rs index 27325438b..7b7f647ef 100644 --- a/crates/sail-common-datafusion/src/utils/items.rs +++ b/crates/sail-common-datafusion/src/utils/items.rs @@ -1,7 +1,7 @@ use std::collections::VecDeque; use std::fmt::Debug; -use datafusion_common::{plan_datafusion_err, plan_err, Result}; +use datafusion_common::{Result, plan_datafusion_err, plan_err}; use either::Either; /// A trait for taking items from a container of expected size. diff --git a/crates/sail-common-datafusion/src/variant.rs b/crates/sail-common-datafusion/src/variant.rs index c4e41ca8e..8a2eaebfe 100644 --- a/crates/sail-common-datafusion/src/variant.rs +++ b/crates/sail-common-datafusion/src/variant.rs @@ -9,7 +9,7 @@ use datafusion::arrow::datatypes::{ DataType, Field, FieldRef, Fields, Schema, SchemaRef, TimeUnit, }; use parquet_variant::Variant; -use parquet_variant_compute::{shred_variant, unshred_variant, VariantArray, VariantType}; +use parquet_variant_compute::{VariantArray, VariantType, shred_variant, unshred_variant}; const MIN_FIELD_FREQUENCY: f64 = 0.10; const MAX_SHREDDED_FIELDS: usize = 300; @@ -786,9 +786,11 @@ mod tests { let DataType::Struct(payload_fields) = payload_field.data_type() else { panic!("expected variant struct"); }; - assert!(payload_fields - .iter() - .any(|field| field.name() == "typed_value")); + assert!( + payload_fields + .iter() + .any(|field| field.name() == "typed_value") + ); let typed_value = payload_fields .iter() .find(|field| field.name() == "typed_value") diff --git a/crates/sail-common/Cargo.toml b/crates/sail-common/Cargo.toml index be8427bdb..7ac30bff9 100644 --- a/crates/sail-common/Cargo.toml +++ b/crates/sail-common/Cargo.toml @@ -28,3 +28,11 @@ iana-time-zone = { workspace = true } tokio = { workspace = true } secrecy = { workspace = true } toml = { workspace = true } +fastrace = { workspace = true } +log = { workspace = true } +tonic = { workspace = true } +tonic-reflection = { workspace = true } +tonic-health = { workspace = true } +tower = { workspace = true } +axum = { workspace = true } +pin-project-lite = { workspace = true } diff --git a/crates/sail-server/src/actor.rs b/crates/sail-common/src/actor.rs similarity index 76% rename from crates/sail-server/src/actor.rs rename to crates/sail-common/src/actor.rs index a07bd5654..afff43597 100644 --- a/crates/sail-server/src/actor.rs +++ b/crates/sail-common/src/actor.rs @@ -1,13 +1,14 @@ use std::fmt::Debug; +use fastrace::Span; use fastrace::future::FutureExt; use fastrace::prelude::SpanContext; -use fastrace::Span; use log::{error, warn}; -use sail_telemetry::common::{SpanAssociation, SpanAttribute, SpanKind}; use tokio::sync::mpsc; use tokio::task::{AbortHandle, JoinSet}; +use crate::telemetry::{SpanAssociation, SpanAttribute, SpanKind}; + const ACTOR_CHANNEL_SIZE: usize = 8; #[tonic::async_trait] @@ -42,6 +43,8 @@ pub struct ActorContext { /// A set of tasks spawned by the actor when processing messages. /// All these tasks will be aborted when the context is dropped. tasks: JoinSet<()>, + /// Actors owned by this actor. All child actors will be aborted when the context is dropped. + children: ActorSystem, } impl ActorContext { @@ -49,6 +52,7 @@ impl ActorContext { Self { handle: handle.clone(), tasks: JoinSet::new(), + children: ActorSystem::new(), } } @@ -83,7 +87,13 @@ impl ActorContext { self.tasks.spawn(task.in_span(span)) } + /// Return the system that owns the children of this actor. + pub fn children_mut(&mut self) -> &mut ActorSystem { + &mut self.children + } + /// Join tasks that have completed. + /// /// Any unhandled errors will be logged here. pub fn reap(&mut self) { while let Some(result) = self.tasks.try_join_next() { @@ -95,6 +105,7 @@ impl ActorContext { } } } + self.children.reap(); } } @@ -132,6 +143,9 @@ impl ActorSystem { /// Wait for all the spawned actors to stop. /// The system can still be used to spawn new actors after this method is called. + /// + /// Please note that the actors must have been sent a stop message before calling this method, + /// otherwise this method will wait indefinitely, causing a deadlock. pub async fn join(&mut self) { while let Some(result) = self.tasks.join_next().await { match result { @@ -142,6 +156,24 @@ impl ActorSystem { } } } + + /// Join actors that have already stopped. + /// + /// Any unhandled errors will be logged here. + pub fn reap(&mut self) { + while let Some(result) = self.tasks.try_join_next() { + match result { + Ok(()) => {} + Err(e) => { + error!("failed to join actor: {e}"); + } + } + } + } + + fn len(&self) -> usize { + self.tasks.len() + } } pub struct ActorHandle { @@ -229,6 +261,10 @@ impl ActorRunner { if n > 0 { warn!("aborting {n} task(s) for {}", T::name()); } + let n = self.ctx.children.len(); + if n > 0 { + warn!("aborting {n} child actor(s) for {}", T::name()); + } } } @@ -248,6 +284,10 @@ mod tests { struct TestActor; + struct ParentActor { + child: Option>>, + } + enum TestMessage { Echo { value: String, @@ -256,6 +296,10 @@ mod tests { Stop, } + enum ParentMessage { + Stop, + } + impl SpanAssociation for TestMessage { fn name(&self) -> Cow<'static, str> { match self { @@ -272,6 +316,18 @@ mod tests { } } + impl SpanAssociation for ParentMessage { + fn name(&self) -> Cow<'static, str> { + match self { + ParentMessage::Stop => "Stop".into(), + } + } + + fn properties(&self) -> impl IntoIterator, Cow<'static, str>)> { + vec![] + } + } + #[tonic::async_trait] impl Actor for TestActor { type Message = TestMessage; @@ -296,6 +352,33 @@ mod tests { } } + #[tonic::async_trait] + impl Actor for ParentActor { + type Message = ParentMessage; + type Options = oneshot::Sender>; + + fn name() -> &'static str { + "ParentActor" + } + + fn new(child: Self::Options) -> Self { + Self { child: Some(child) } + } + + async fn start(&mut self, ctx: &mut ActorContext) { + let child = ctx.children_mut().spawn::(()); + if let Some(sender) = self.child.take() { + let _ = sender.send(child); + } + } + + fn receive(&mut self, _: &mut ActorContext, message: Self::Message) -> ActorAction { + match message { + ParentMessage::Stop => ActorAction::Stop, + } + } + } + #[tokio::test] async fn test_actor_handle_send() { let mut system = ActorSystem::new(); @@ -320,4 +403,22 @@ mod tests { assert!(matches!(result, Ok(()))); system.join().await; } + + #[tokio::test] + async fn test_child_actor_stops_with_parent() { + let mut system = ActorSystem::new(); + let (tx, rx) = oneshot::channel(); + let parent = system.spawn::(tx); + let child = rx.await; + assert!(child.is_ok()); + let Ok(child) = child else { + return; + }; + + assert!(parent.send(ParentMessage::Stop).await.is_ok()); + system.join().await; + + child.sender.closed().await; + assert!(child.sender.is_closed()); + } } diff --git a/crates/sail-common/src/config/application.rs b/crates/sail-common/src/config/application.rs index c8bcd4da3..825af1975 100644 --- a/crates/sail-common/src/config/application.rs +++ b/crates/sail-common/src/config/application.rs @@ -5,7 +5,7 @@ use secrecy::SecretString; use serde::{Deserialize, Serialize}; use crate::config::loader::{ - deserialize_non_empty_string, deserialize_non_zero, deserialize_unknown_unit, ConfigDefinition, + ConfigDefinition, deserialize_non_empty_string, deserialize_non_zero, deserialize_unknown_unit, }; use crate::config::observer::{ serialize_non_empty_string, serialize_non_zero, serialize_optional_secret, @@ -38,6 +38,16 @@ pub struct AppConfig { pub internal: (), } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointConfig { + #[serde( + serialize_with = "serialize_non_empty_string", + deserialize_with = "deserialize_non_empty_string" + )] + pub path: Option, +} + /// A configuration provider that injects placeholder internal configuration. struct InternalConfigPlaceholder; @@ -176,11 +186,15 @@ pub struct TemporaryFilesConfig { #[serde(deny_unknown_fields)] pub struct ClusterConfig { pub enable_tls: bool, + #[serde(skip_serializing)] + pub session_id: String, pub driver_listen_host: String, pub driver_listen_port: u16, pub driver_external_host: String, pub driver_external_port: u16, #[serde(skip_serializing)] + pub driver_id: u64, + #[serde(skip_serializing)] pub worker_id: u64, pub worker_listen_host: String, pub worker_listen_port: u16, @@ -198,6 +212,7 @@ pub struct ClusterConfig { pub task_stream_creation_timeout_secs: u64, pub task_max_attempts: usize, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackend, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -282,6 +297,82 @@ mod retry_strategy { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde( + into = "shuffle_backend::ShuffleBackend", + from = "shuffle_backend::ShuffleBackend" +)] +pub enum ShuffleBackend { + Flight, + Storage(StorageShuffleBackend), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StorageShuffleBackend { + #[serde( + serialize_with = "serialize_non_empty_string", + deserialize_with = "deserialize_non_empty_string" + )] + pub path: Option, + pub max_file_size: usize, + pub compression: ShuffleCompression, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ShuffleCompression { + None, + Lz4, + Zstd, +} + +mod shuffle_backend { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum Type { + Flight, + Storage, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct ShuffleBackend { + pub r#type: Type, + pub storage: super::StorageShuffleBackend, + } + + impl From for super::ShuffleBackend { + fn from(value: ShuffleBackend) -> Self { + match value.r#type { + Type::Flight => super::ShuffleBackend::Flight, + Type::Storage => super::ShuffleBackend::Storage(value.storage), + } + } + } + + impl From for ShuffleBackend { + fn from(value: super::ShuffleBackend) -> Self { + match value { + super::ShuffleBackend::Flight => ShuffleBackend { + r#type: Type::Flight, + storage: super::StorageShuffleBackend { + path: None, + max_file_size: 0, + compression: super::ShuffleCompression::None, + }, + }, + super::ShuffleBackend::Storage(storage) => ShuffleBackend { + r#type: Type::Storage, + storage, + }, + } + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ExecutionConfig { @@ -290,6 +381,7 @@ pub struct ExecutionConfig { pub collect_statistics: bool, pub use_row_number_estimates_to_optimize_partitioning: bool, pub file_listing_cache: FileListingCacheConfig, + pub checkpoint: CheckpointConfig, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -509,6 +601,14 @@ pub struct OptimizerConfig { pub expand_views_at_output: bool, } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OneLakeApi { + Delta, + #[default] + Iceberg, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(tag = "type", rename_all = "snake_case")] @@ -541,6 +641,14 @@ pub enum CatalogType { serialize_with = "serialize_optional_secret" )] bearer_access_token: Option, + /// Path to a file holding the bearer token. When set, the token is + /// re-read from this file for every request, so a rotated token (for + /// example a kubelet-projected service account token) is picked up + /// without restarting the server. Takes precedence over + /// `bearer_access_token`. The path is not a secret, so it is kept as a + /// plain string. + #[serde(skip_serializing_if = "Option::is_none")] + bearer_access_token_file: Option, #[serde(flatten)] cache: CatalogCacheConfig, }, @@ -562,6 +670,8 @@ pub enum CatalogType { OneLake { name: String, url: String, + #[serde(default)] + api: OneLakeApi, #[serde( skip_serializing_if = "Option::is_none", serialize_with = "serialize_optional_secret" @@ -573,6 +683,8 @@ pub enum CatalogType { Glue { name: String, #[serde(skip_serializing_if = "Option::is_none")] + catalog_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] region: Option, #[serde(skip_serializing_if = "Option::is_none")] endpoint_url: Option, @@ -620,9 +732,7 @@ pub struct TelemetryConfig { pub export_traces: bool, pub export_metrics: bool, pub export_logs: bool, - pub otlp_endpoint: String, - pub otlp_protocol: OtlpProtocol, - pub otlp_timeout_secs: u64, + pub exporter: TelemetryExporterConfig, pub traces_export_interval_secs: u64, pub metrics_export_interval_secs: u64, pub metrics_collection_interval_secs: u64, @@ -631,6 +741,24 @@ pub struct TelemetryConfig { pub logs_export_batch_size: u64, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TelemetryExporterConfig { + pub otlp: OtlpConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OtlpConfig { + #[serde( + serialize_with = "serialize_non_empty_string", + deserialize_with = "deserialize_non_empty_string" + )] + pub endpoint: Option, + pub protocol: OtlpProtocol, + pub timeout_secs: u64, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OtlpProtocol { @@ -653,8 +781,10 @@ macro_rules! define_cluster_config_env { impl ClusterConfigEnv { define_cluster_config_env! { ENABLE_TLS, + SESSION_ID, DRIVER_EXTERNAL_HOST, DRIVER_EXTERNAL_PORT, + DRIVER_ID, WORKER_ID, WORKER_LISTEN_HOST, WORKER_EXTERNAL_HOST, @@ -662,5 +792,9 @@ impl ClusterConfigEnv { TASK_STREAM_BUFFER, TASK_STREAM_CREATION_TIMEOUT_SECS, RPC_RETRY_STRATEGY, + SHUFFLE_BACKEND__TYPE, + SHUFFLE_BACKEND__STORAGE__PATH, + SHUFFLE_BACKEND__STORAGE__MAX_FILE_SIZE, + SHUFFLE_BACKEND__STORAGE__COMPRESSION, } } diff --git a/crates/sail-common/src/config/application.yaml b/crates/sail-common/src/config/application.yaml index 9132bc3b9..30a8d97a3 100644 --- a/crates/sail-common/src/config/application.yaml +++ b/crates/sail-common/src/config/application.yaml @@ -71,6 +71,12 @@ description: Whether to enable TLS for cluster communication. experimental: true +- key: cluster.session_id + type: string + default: "" + description: + hidden: true + - key: cluster.driver_listen_host type: string default: "127.0.0.1" @@ -96,6 +102,12 @@ If the value is `0`, the port is assumed to be the same as the port on which the driver listens. +- key: cluster.driver_id + type: number + default: "0" + description: + hidden: true + - key: cluster.worker_id type: number default: "0" @@ -244,6 +256,40 @@ when using the `exponential_backoff` retry strategy. experimental: true +- key: cluster.shuffle_backend.type + type: string + default: "flight" + description: | + The shuffle backend type. Valid values are `flight` and `storage`. + The `flight` shuffle backend streams shuffle data directly among tasks via Arrow Flight. + The `storage` shuffle backend materializes merged shuffle partitions in object storage + before running dependent stages. + experimental: true + +- key: cluster.shuffle_backend.storage.path + type: string + default: "" + description: | + The object storage base URL for storage-backed shuffle data. + This is required when `cluster.shuffle_backend.type` is `storage`. + experimental: true + +- key: cluster.shuffle_backend.storage.max_file_size + type: number + default: "134217728" + description: | + The maximum size in bytes of each storage-backed shuffle file. Larger partitions are + split into multiple Arrow IPC files. + experimental: true + +- key: cluster.shuffle_backend.storage.compression + type: string + default: "zstd" + description: | + Arrow IPC compression for storage-backed shuffle files. Valid values are `none`, + `lz4`, and `zstd`. + experimental: true + - key: execution.batch_size type: number default: "8192" @@ -311,6 +357,17 @@ This setting can only be configured at startup and cannot be changed at runtime. experimental: true +- key: execution.checkpoint.path + type: string + default: "" + description: | + The object storage base path for DataFrame checkpoints managed by the server. The path must be + readable and writable by the driver and every worker. Checkpoint operations are rejected when + this value is empty. Sail does not fall back to driver memory or local worker storage. Sail + removes each session namespace on shutdown. An object storage lifecycle policy can clean up + data left by an ungraceful process exit. + experimental: true + - key: parquet.enable_page_index type: boolean default: "true" @@ -799,13 +856,15 @@ description: Whether to export logs to an OpenTelemetry collector. experimental: true -- key: telemetry.otlp_endpoint +- key: telemetry.exporter.otlp.endpoint type: string - default: "http://localhost:4317" - description: The OTLP endpoint for telemetry data export. + default: "" + description: | + The OTLP endpoint for telemetry data export. + The OTLP exporter is disabled unless this option is set. experimental: true -- key: telemetry.otlp_protocol +- key: telemetry.exporter.otlp.protocol type: string default: "grpc" description: | @@ -813,7 +872,7 @@ Valid values are `grpc`, `http-binary`, and `http-json`. experimental: true -- key: telemetry.otlp_timeout_secs +- key: telemetry.exporter.otlp.timeout_secs type: number default: "10" description: The timeout in seconds for OTLP telemetry data export. diff --git a/crates/sail-common/src/config/cli.rs b/crates/sail-common/src/config/cli.rs index 2e18461b2..7fc68046e 100644 --- a/crates/sail-common/src/config/cli.rs +++ b/crates/sail-common/src/config/cli.rs @@ -1,5 +1,5 @@ -use figment::providers::{Env, Serialized}; use figment::Figment; +use figment::providers::{Env, Serialized}; use serde::{Deserialize, Serialize}; use crate::error::{CommonError, CommonResult}; diff --git a/crates/sail-common/src/lib.rs b/crates/sail-common/src/lib.rs index c25236311..5b2d68fe2 100644 --- a/crates/sail-common/src/lib.rs +++ b/crates/sail-common/src/lib.rs @@ -1,10 +1,10 @@ +pub mod actor; pub mod config; -pub mod datetime; -pub mod debug; pub mod error; pub mod geoarrow; -pub mod object; pub mod runtime; +pub mod server; pub mod spec; -pub mod string; +pub mod telemetry; pub mod tests; +pub mod utils; diff --git a/crates/sail-server/src/builder.rs b/crates/sail-common/src/server/builder.rs similarity index 97% rename from crates/sail-server/src/builder.rs rename to crates/sail-common/src/server/builder.rs index 427227194..95df6c93c 100644 --- a/crates/sail-server/src/builder.rs +++ b/crates/sail-common/src/server/builder.rs @@ -1,16 +1,17 @@ use std::convert::Infallible; use std::future::Future; -use sail_telemetry::layers::TracingServerLayer; use tokio::net::TcpListener; use tonic::body::Body; -use tonic::codegen::http::Request; use tonic::codegen::Service; +use tonic::codegen::http::Request; use tonic::server::NamedService; use tonic::transport::server::{Router, TcpIncoming}; use tonic_health::server::HealthReporter; -use tower::layer::util::{Identity, Stack}; use tower::ServiceBuilder; +use tower::layer::util::{Identity, Stack}; + +use crate::telemetry::TracingServerLayer; pub struct ServerBuilderOptions { pub nodelay: bool, @@ -25,8 +26,8 @@ impl Default for ServerBuilderOptions { Self { // Disables Nagle's algorithm nodelay: true, - keepalive: Some(std::time::Duration::from_secs(60)), - http2_keepalive_interval: Some(std::time::Duration::from_secs(60)), + keepalive: Some(std::time::Duration::from_mins(1)), + http2_keepalive_interval: Some(std::time::Duration::from_mins(1)), http2_keepalive_timeout: Some(std::time::Duration::from_secs(10)), http2_adaptive_window: Some(true), } diff --git a/crates/sail-server/src/lib.rs b/crates/sail-common/src/server/mod.rs similarity index 55% rename from crates/sail-server/src/lib.rs rename to crates/sail-common/src/server/mod.rs index 9471e84dc..391f59bba 100644 --- a/crates/sail-server/src/lib.rs +++ b/crates/sail-common/src/server/mod.rs @@ -1,6 +1,3 @@ -pub mod actor; mod builder; -mod retry; pub use builder::{ServerBuilder, ServerBuilderOptions}; -pub use retry::RetryStrategy; diff --git a/crates/sail-common/src/spec/plan.rs b/crates/sail-common/src/spec/plan.rs index ec6df9d71..052a2166c 100644 --- a/crates/sail-common/src/spec/plan.rs +++ b/crates/sail-common/src/spec/plan.rs @@ -290,6 +290,13 @@ pub enum CommandNode { HtmlString(HtmlString), // TODO: add all the "analyze" requests // TODO: should streaming query request be added here? + /// A command wrapped with referenced subquery plans (similar to QueryNode::WithRelations). + /// This is used when a command references DataFrames passed as named arguments + /// (e.g., `spark.sql("MERGE INTO t USING {df}", df=dataframe)`). + WithRelations { + root: Box, + references: Vec, + }, // catalog operations CurrentDatabase, SetCurrentDatabase { @@ -310,6 +317,12 @@ pub enum CommandNode { database: Option, pattern: String, }, + ShowFunctions { + database: Option, + pattern: Option, + show_user_functions: bool, + show_system_functions: bool, + }, ListTables { database: Option, pattern: Option, @@ -434,6 +447,10 @@ pub enum CommandNode { is_global: bool, if_exists: bool, }, + RemoteCheckpoint { + relation_id: String, + input: Box, + }, Write(Write), WriteTo(WriteTo), WriteStream(WriteStream), @@ -765,9 +782,9 @@ pub struct ShowString { #[serde(rename_all = "camelCase")] pub struct Pivot { pub input: Box, - /// The group-by columns for the pivot operation (only supported in the DataFrame API). - /// When the list is empty (for SQL statements), all the remaining columns are included. - pub grouping: Vec, + /// The group-by columns for the pivot operation, set for the DataFrame API (possibly + /// empty). When `None` (for SQL statements), all the remaining columns are included. + pub grouping: Option>, pub aggregate: Vec, pub columns: Vec, pub values: Vec, diff --git a/crates/sail-telemetry/src/common.rs b/crates/sail-common/src/telemetry/common.rs similarity index 97% rename from crates/sail-telemetry/src/common.rs rename to crates/sail-common/src/telemetry/common.rs index df9bd34c6..0a4547f79 100644 --- a/crates/sail-telemetry/src/common.rs +++ b/crates/sail-common/src/telemetry/common.rs @@ -41,6 +41,7 @@ impl SpanAttribute { pub const EXCEPTION_MESSAGE: &'static str = "exception.message"; pub const EXCEPTION_TYPE: &'static str = "exception.type"; pub const RETRY_ATTEMPT: &'static str = "retry.attempt"; + pub const CLUSTER_DRIVER_ID: &'static str = "cluster.driver.id"; pub const CLUSTER_DRIVER_PORT: &'static str = "cluster.driver.port"; pub const CLUSTER_WORKER_ID: &'static str = "cluster.worker.id"; pub const CLUSTER_WORKER_HOST: &'static str = "cluster.worker.host"; @@ -54,7 +55,6 @@ impl SpanAttribute { pub const EXECUTION_TASK_MESSAGE: &'static str = "execution.task.message"; pub const EXECUTION_TASK_ERROR_CAUSE: &'static str = "execution.task.error_cause"; pub const EXECUTION_STREAM_LOCAL_STORAGE: &'static str = "execution.stream.local.storage"; - pub const EXECUTION_STREAM_REMOTE_URI: &'static str = "execution.stream.remote.uri"; pub const SESSION_ID: &'static str = "session.id"; } diff --git a/crates/sail-telemetry/src/futures.rs b/crates/sail-common/src/telemetry/futures.rs similarity index 100% rename from crates/sail-telemetry/src/futures.rs rename to crates/sail-common/src/telemetry/futures.rs diff --git a/crates/sail-telemetry/src/layers/client.rs b/crates/sail-common/src/telemetry/layers/client.rs similarity index 72% rename from crates/sail-telemetry/src/layers/client.rs rename to crates/sail-common/src/telemetry/layers/client.rs index d35fc40e5..cc0a0611a 100644 --- a/crates/sail-telemetry/src/layers/client.rs +++ b/crates/sail-common/src/telemetry/layers/client.rs @@ -1,14 +1,13 @@ use std::task::{Context, Poll}; -use datafusion::object_store::HeaderValue; +use fastrace::Span; use fastrace::collector::SpanContext; use fastrace::future::{FutureExt, InSpan}; -use fastrace::Span; -use tonic::codegen::http::Request; use tonic::codegen::Service; +use tonic::codegen::http::{HeaderValue, Request}; use tower::Layer; -use crate::common::{ContextPropagationHeader, SpanAttribute, SpanKind}; +use super::super::common::{ContextPropagationHeader, SpanAttribute, SpanKind}; #[derive(Clone)] pub struct TracingClientLayer; @@ -41,11 +40,11 @@ where fn call(&mut self, mut req: Request) -> Self::Future { let span = Span::enter_with_local_parent(format!("{} {}", req.method(), req.uri().path())) .with_properties(|| [(SpanAttribute::SPAN_KIND, SpanKind::CLIENT)]); - if let Some(current) = SpanContext::from_span(&span) { - if let Ok(value) = HeaderValue::from_str(¤t.encode_w3c_traceparent()) { - req.headers_mut() - .insert(ContextPropagationHeader::TRACEPARENT, value); - } + if let Some(current) = SpanContext::from_span(&span) + && let Ok(value) = HeaderValue::from_str(¤t.encode_w3c_traceparent()) + { + req.headers_mut() + .insert(ContextPropagationHeader::TRACEPARENT, value); } self.inner.call(req).in_span(span) diff --git a/crates/sail-telemetry/src/layers/mod.rs b/crates/sail-common/src/telemetry/layers/mod.rs similarity index 100% rename from crates/sail-telemetry/src/layers/mod.rs rename to crates/sail-common/src/telemetry/layers/mod.rs diff --git a/crates/sail-telemetry/src/layers/server.rs b/crates/sail-common/src/telemetry/layers/server.rs similarity index 94% rename from crates/sail-telemetry/src/layers/server.rs rename to crates/sail-common/src/telemetry/layers/server.rs index d9b341b52..f5d78e696 100644 --- a/crates/sail-telemetry/src/layers/server.rs +++ b/crates/sail-common/src/telemetry/layers/server.rs @@ -1,12 +1,12 @@ use std::task::{Context, Poll}; +use fastrace::Span; use fastrace::collector::SpanContext; use fastrace::future::{FutureExt, InSpan}; -use fastrace::Span; use tonic::codegen::http::Request; use tower::{Layer, Service}; -use crate::common::{ContextPropagationHeader, SpanAttribute, SpanKind}; +use super::super::common::{ContextPropagationHeader, SpanAttribute, SpanKind}; #[derive(Clone)] pub struct TracingServerLayer; diff --git a/crates/sail-common/src/telemetry/mod.rs b/crates/sail-common/src/telemetry/mod.rs new file mode 100644 index 000000000..3fd26c690 --- /dev/null +++ b/crates/sail-common/src/telemetry/mod.rs @@ -0,0 +1,9 @@ +mod common; +mod futures; +mod layers; +mod recorder; + +pub use common::*; +pub use futures::*; +pub use layers::*; +pub use recorder::*; diff --git a/crates/sail-telemetry/src/recorder.rs b/crates/sail-common/src/telemetry/recorder.rs similarity index 90% rename from crates/sail-telemetry/src/recorder.rs rename to crates/sail-common/src/telemetry/recorder.rs index 68c22f8da..817d91621 100644 --- a/crates/sail-telemetry/src/recorder.rs +++ b/crates/sail-common/src/telemetry/recorder.rs @@ -2,7 +2,7 @@ use core::fmt; use fastrace::Span; -use crate::common::SpanAttribute; +use super::common::SpanAttribute; /// Records the error message from a [`Result`] in the given span. pub fn record_error(span: &Span, result: &Result) diff --git a/crates/sail-common/src/datetime.rs b/crates/sail-common/src/utils/datetime.rs similarity index 100% rename from crates/sail-common/src/datetime.rs rename to crates/sail-common/src/utils/datetime.rs diff --git a/crates/sail-common/src/debug.rs b/crates/sail-common/src/utils/debug.rs similarity index 100% rename from crates/sail-common/src/debug.rs rename to crates/sail-common/src/utils/debug.rs diff --git a/crates/sail-common/src/utils/http.rs b/crates/sail-common/src/utils/http.rs new file mode 100644 index 000000000..d3ba67ab1 --- /dev/null +++ b/crates/sail-common/src/utils/http.rs @@ -0,0 +1,7 @@ +/// User agent for Sail requests to remote services. +/// +/// References: +/// * +/// * (obsolete) +/// * +pub const SAIL_USER_AGENT: &str = concat!("Sail/", env!("CARGO_PKG_VERSION")); diff --git a/crates/sail-common/src/utils/mod.rs b/crates/sail-common/src/utils/mod.rs new file mode 100644 index 000000000..50ee55928 --- /dev/null +++ b/crates/sail-common/src/utils/mod.rs @@ -0,0 +1,6 @@ +pub mod datetime; +pub mod debug; +pub mod http; +pub mod object; +pub mod retry; +pub mod string; diff --git a/crates/sail-common/src/object.rs b/crates/sail-common/src/utils/object.rs similarity index 100% rename from crates/sail-common/src/object.rs rename to crates/sail-common/src/utils/object.rs diff --git a/crates/sail-server/src/retry.rs b/crates/sail-common/src/utils/retry.rs similarity index 95% rename from crates/sail-server/src/retry.rs rename to crates/sail-common/src/utils/retry.rs index 31f4d9dd8..e05945a46 100644 --- a/crates/sail-server/src/retry.rs +++ b/crates/sail-common/src/utils/retry.rs @@ -3,10 +3,9 @@ use std::time::Duration; use fastrace::Span; use log::warn; -use sail_common::config; -use sail_telemetry::common::SpanAttribute; -use sail_telemetry::futures::TracingFutureExt; -use sail_telemetry::recorder::record_error; + +use crate::config; +use crate::telemetry::{SpanAttribute, TracingFutureExt, record_error}; #[derive(Debug, Clone)] pub enum RetryStrategy { diff --git a/crates/sail-common/src/string.rs b/crates/sail-common/src/utils/string.rs similarity index 100% rename from crates/sail-common/src/string.rs rename to crates/sail-common/src/utils/string.rs diff --git a/crates/sail-data-source/data/options/csv.yaml b/crates/sail-data-source/data/options/csv.yaml index 53ea61f64..dba809dce 100644 --- a/crates/sail-data-source/data/options/csv.yaml +++ b/crates/sail-data-source/data/options/csv.yaml @@ -1,6 +1,28 @@ # References: # - [1] https://spark.apache.org/docs/4.0.0/sql-data-sources-csv.html#data-source-option +- key: path_glob_filter + description: | + A glob pattern to only include files with file names matching the pattern. + default: + value: "" + parser: crate::options::parsers::parse_optional_string + supported: true + rust_type: "Option" + scopes: + - read + origins: + - type: option + keys: + - path_glob_filter + - pathGlobFilter + parser: crate::options::parsers::parse_optional_string + - type: table_property + keys: + - option.path_glob_filter + - option.pathGlobFilter + parser: crate::options::parsers::parse_optional_string + # TODO: Delimiter can be one or more characters - key: delimiter description: | diff --git a/crates/sail-data-source/data/options/json.yaml b/crates/sail-data-source/data/options/json.yaml index e316a56db..7b14d5abc 100644 --- a/crates/sail-data-source/data/options/json.yaml +++ b/crates/sail-data-source/data/options/json.yaml @@ -1,6 +1,28 @@ # References: # - [1] https://spark.apache.org/docs/4.0.0/sql-data-sources-json.html#data-source-option +- key: path_glob_filter + description: | + A glob pattern to only include files with file names matching the pattern. + default: + value: "" + parser: crate::options::parsers::parse_optional_string + supported: true + rust_type: "Option" + scopes: + - read + origins: + - type: option + keys: + - path_glob_filter + - pathGlobFilter + parser: crate::options::parsers::parse_optional_string + - type: table_property + keys: + - option.path_glob_filter + - option.pathGlobFilter + parser: crate::options::parsers::parse_optional_string + - key: schema_infer_max_records description: | The maximum number of rows to read from JSON files for schema inference if needed. diff --git a/crates/sail-data-source/data/options/parquet.yaml b/crates/sail-data-source/data/options/parquet.yaml index 4790b5dd4..9bbb55cd6 100644 --- a/crates/sail-data-source/data/options/parquet.yaml +++ b/crates/sail-data-source/data/options/parquet.yaml @@ -6,30 +6,27 @@ # TODO: Spark global: https://spark.apache.org/docs/4.0.0/sql-data-sources-parquet.html#configuration # TODO: DataFusion `column_specific_options` and `key_value_metadata` -- key: extension +- key: path_glob_filter description: | - (Reading) The file extension used to filter files when listing a - directory of Parquet files. Defaults to `.parquet`. Setting this to - an empty string disables extension filtering entirely, which allows - reading Parquet files written under non-standard suffixes (e.g. `.hive`). + A glob pattern to only include files with file names matching the pattern. default: - value: ".parquet" - parser: crate::options::parsers::parse_string + value: "" + parser: crate::options::parsers::parse_optional_string supported: true - rust_type: String + rust_type: "Option" scopes: - read origins: - type: option keys: - - extension - - fileExtension - parser: crate::options::parsers::parse_string + - path_glob_filter + - pathGlobFilter + parser: crate::options::parsers::parse_optional_string - type: table_property keys: - - option.extension - - option.fileExtension - parser: crate::options::parsers::parse_string + - option.path_glob_filter + - option.pathGlobFilter + parser: crate::options::parsers::parse_optional_string - key: enable_page_index description: | diff --git a/crates/sail-data-source/data/options/text.yaml b/crates/sail-data-source/data/options/text.yaml index 8c4ceabd0..70674e682 100644 --- a/crates/sail-data-source/data/options/text.yaml +++ b/crates/sail-data-source/data/options/text.yaml @@ -1,6 +1,28 @@ # References: # - https://spark.apache.org/docs/latest/sql-data-sources-text.html +- key: path_glob_filter + description: | + A glob pattern to only include files with file names matching the pattern. + default: + value: "" + parser: crate::options::parsers::parse_optional_string + supported: true + rust_type: "Option" + scopes: + - read + origins: + - type: option + keys: + - path_glob_filter + - pathGlobFilter + parser: crate::options::parsers::parse_optional_string + - type: table_property + keys: + - option.path_glob_filter + - option.pathGlobFilter + parser: crate::options::parsers::parse_optional_string + - key: whole_text description: | If true, read each file from input path(s) as a single row. diff --git a/crates/sail-data-source/src/formats/arrow/read.rs b/crates/sail-data-source/src/formats/arrow/read.rs index c91342cd5..34f0ce63d 100644 --- a/crates/sail-data-source/src/formats/arrow/read.rs +++ b/crates/sail-data-source/src/formats/arrow/read.rs @@ -9,10 +9,10 @@ use datafusion::arrow::ipc::root_as_message; use datafusion::catalog::Session; use datafusion::datasource::physical_plan::ArrowSource; use datafusion_common::parsers::CompressionTypeVariant; -use datafusion_common::{internal_datafusion_err, Result}; +use datafusion_common::{Result, internal_datafusion_err}; use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; -use futures::stream::BoxStream; use futures::StreamExt; +use futures::stream::BoxStream; use object_store::path::Path; use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore, ObjectStoreExt}; @@ -21,7 +21,7 @@ use crate::listing::source::{ListingFileSample, ListingScanInput, ReadFormat}; #[derive(Debug, Default, Clone)] pub struct ArrowReadFormat; -const ARROW_MAGIC: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'1']; +const ARROW_MAGIC: [u8; 6] = *b"ARROW1"; const CONTINUATION_MARKER: [u8; 4] = [0xff; 4]; async fn is_object_in_arrow_ipc_file_format( @@ -74,8 +74,8 @@ impl ReadFormat for ArrowReadFormat { Ok(reader) => reader.schema(), Err(stream_error) => { return Err(internal_datafusion_err!( - "Failed to parse Arrow file as either file format or stream format. File format error: {file_error}. Stream format error: {stream_error}" - )); + "Failed to parse Arrow file as either file format or stream format. File format error: {file_error}. Stream format error: {stream_error}" + )); } } } diff --git a/crates/sail-data-source/src/formats/avro/read.rs b/crates/sail-data-source/src/formats/avro/read.rs index a4d1c23ed..021717eff 100644 --- a/crates/sail-data-source/src/formats/avro/read.rs +++ b/crates/sail-data-source/src/formats/avro/read.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{Schema, SchemaRef}; use datafusion::catalog::Session; use datafusion::datasource::physical_plan::AvroSource; -use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::Result; +use datafusion_common::parsers::CompressionTypeVariant; use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; use datafusion_datasource_avro::read_avro_schema_from_reader; use object_store::{GetResultPayload, ObjectStoreExt}; diff --git a/crates/sail-data-source/src/formats/binary/mod.rs b/crates/sail-data-source/src/formats/binary/mod.rs index 09a200c32..9303cfb5e 100644 --- a/crates/sail-data-source/src/formats/binary/mod.rs +++ b/crates/sail-data-source/src/formats/binary/mod.rs @@ -3,12 +3,12 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datafusion::catalog::Session; use datafusion_common::arrow::datatypes::SchemaRef; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use sail_common_datafusion::datasource::OptionLayer; use crate::listing::source::{FormatFactory, ListingTableFormat}; -use crate::options::gen::BinaryReadOptions; use crate::options::ResolveOptions; +use crate::options::r#gen::BinaryReadOptions; pub mod options; mod read; @@ -19,11 +19,6 @@ mod write; pub use read::BinaryReadFormat; pub use write::BinaryWriteFormat; -#[derive(Default, Debug, Clone, PartialEq)] -pub struct TableBinaryOptions { - pub path_glob_filter: Option, -} - pub type BinaryTableFormat = ListingTableFormat; #[derive(Debug, Default)] diff --git a/crates/sail-data-source/src/formats/binary/options.rs b/crates/sail-data-source/src/formats/binary/options.rs index f79c1112b..f896850d5 100644 --- a/crates/sail-data-source/src/formats/binary/options.rs +++ b/crates/sail-data-source/src/formats/binary/options.rs @@ -2,17 +2,9 @@ use datafusion::catalog::Session; use sail_common_datafusion::datasource::OptionLayer; use crate::error::DataSourceResult; -use crate::formats::binary::TableBinaryOptions; -use crate::options::gen::{BinaryReadOptions, BinaryReadPartialOptions}; +use crate::options::r#gen::{BinaryReadOptions, BinaryReadPartialOptions}; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; -impl BinaryReadOptions { - pub fn into_table_options(self) -> TableBinaryOptions { - let BinaryReadOptions { path_glob_filter } = self; - TableBinaryOptions { path_glob_filter } - } -} - impl ResolveOptions for BinaryReadOptions { fn resolve(_ctx: &dyn Session, options: Vec) -> DataSourceResult { let mut partial = BinaryReadPartialOptions::initialize(); @@ -27,8 +19,8 @@ impl ResolveOptions for BinaryReadOptions { mod tests { use datafusion::prelude::SessionContext; - use crate::options::gen::BinaryReadOptions; - use crate::options::{option_list, ResolveOptions}; + use crate::options::r#gen::BinaryReadOptions; + use crate::options::{ResolveOptions, option_list}; #[test] fn test_resolve_binary_read_options() -> datafusion_common::Result<()> { @@ -37,20 +29,17 @@ mod tests { let kv = option_list(&[]); let options = BinaryReadOptions::resolve(&state, vec![kv]) - .map_err(datafusion_common::DataFusionError::from)? - .into_table_options(); + .map_err(datafusion_common::DataFusionError::from)?; assert_eq!(options.path_glob_filter, None); let kv = option_list(&[("path_glob_filter", "*.png")]); let options = BinaryReadOptions::resolve(&state, vec![kv]) - .map_err(datafusion_common::DataFusionError::from)? - .into_table_options(); + .map_err(datafusion_common::DataFusionError::from)?; assert_eq!(options.path_glob_filter, Some("*.png".to_string())); let kv = option_list(&[("pathGlobFilter", "*.pdf")]); let options = BinaryReadOptions::resolve(&state, vec![kv]) - .map_err(datafusion_common::DataFusionError::from)? - .into_table_options(); + .map_err(datafusion_common::DataFusionError::from)?; assert_eq!(options.path_glob_filter, Some("*.pdf".to_string())); Ok(()) diff --git a/crates/sail-data-source/src/formats/binary/read.rs b/crates/sail-data-source/src/formats/binary/read.rs index 53d366aff..3c85159f9 100644 --- a/crates/sail-data-source/src/formats/binary/read.rs +++ b/crates/sail-data-source/src/formats/binary/read.rs @@ -2,13 +2,13 @@ use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::Session; -use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::Result; +use datafusion_common::parsers::CompressionTypeVariant; use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; use crate::formats::binary::source::BinarySource; use crate::listing::source::{ListingFileSample, ListingScanInput, ReadFormat}; -use crate::options::gen::BinaryReadOptions; +use crate::options::r#gen::BinaryReadOptions; #[derive(Debug, Clone)] pub struct BinaryReadFormat { @@ -43,12 +43,7 @@ impl ReadFormat for BinaryReadFormat { } async fn scan(&self, _ctx: &dyn Session, input: ListingScanInput) -> Result { - let options = self.options.clone().into_table_options(); - - let file_source = Arc::new(BinarySource::new( - input.schema, - options.path_glob_filter.clone(), - )); + let file_source = Arc::new(BinarySource::new(input.schema)); let config = FileScanConfigBuilder::new(input.object_store_url, file_source) .with_file_groups(input.file_groups) @@ -63,4 +58,8 @@ impl ReadFormat for BinaryReadFormat { Ok(config) } + + fn path_glob_filter(&self) -> Option<&str> { + self.options.path_glob_filter.as_deref() + } } diff --git a/crates/sail-data-source/src/formats/binary/source.rs b/crates/sail-data-source/src/formats/binary/source.rs index 8f0fdd4e6..558ace505 100644 --- a/crates/sail-data-source/src/formats/binary/source.rs +++ b/crates/sail-data-source/src/formats/binary/source.rs @@ -16,32 +16,21 @@ use crate::formats::binary::reader::{BinaryFileMetadata, BinaryFileReader}; #[derive(Debug, Clone)] pub struct BinarySource { table_schema: TableSchema, - path_glob_filter: Option, batch_size: Option, metrics: ExecutionPlanMetricsSet, projection: SplitProjection, } impl BinarySource { - pub fn new(table_schema: impl Into, path_glob_filter: Option) -> Self { + pub fn new(table_schema: impl Into) -> Self { let table_schema = table_schema.into(); Self { projection: SplitProjection::unprojected(&table_schema), table_schema, - path_glob_filter, batch_size: None, metrics: ExecutionPlanMetricsSet::new(), } } - - pub fn with_path_glob_filter(mut self, path_glob_filter: Option) -> Self { - self.path_glob_filter = path_glob_filter; - self - } - - pub fn path_glob_filter(&self) -> Option<&String> { - self.path_glob_filter.as_ref() - } } impl From for Arc { @@ -118,17 +107,6 @@ impl BinaryOpener { impl FileOpener for BinaryOpener { fn open(&self, file: PartitionedFile) -> Result { - if let Some(ref glob_pattern) = self.config.path_glob_filter { - let file_name = file.object_meta.location.filename().unwrap_or(""); - let pattern = glob::Pattern::new(glob_pattern) - .map_err(|e| DataFusionError::External(Box::new(e)))?; - if !pattern.matches(file_name) { - return Ok(Box::pin( - async move { Ok(futures::stream::empty().boxed()) }, - )); - } - } - let store = Arc::clone(&self.object_store); let location = file.object_meta.location.clone(); let last_modified = file.object_meta.last_modified; diff --git a/crates/sail-data-source/src/formats/binary/write.rs b/crates/sail-data-source/src/formats/binary/write.rs index 1b18f8959..3f98a8772 100644 --- a/crates/sail-data-source/src/formats/binary/write.rs +++ b/crates/sail-data-source/src/formats/binary/write.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::catalog::Session; use datafusion::physical_plan::ExecutionPlan; -use datafusion_common::{not_impl_err, Result}; +use datafusion_common::{Result, not_impl_err}; use crate::listing::source::{ListingSinkInput, WriteFormat}; diff --git a/crates/sail-data-source/src/formats/console/mod.rs b/crates/sail-data-source/src/formats/console/mod.rs index 6e42f669a..740075853 100644 --- a/crates/sail-data-source/src/formats/console/mod.rs +++ b/crates/sail-data-source/src/formats/console/mod.rs @@ -9,7 +9,7 @@ use datafusion::execution::SessionState; use datafusion::logical_expr::{Extension, LogicalPlan, TableSource, UserDefinedLogicalNode}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; -use datafusion_common::{internal_err, not_impl_err, plan_err, DFSchema, DFSchemaRef, Result}; +use datafusion_common::{DFSchema, DFSchemaRef, Result, internal_err, not_impl_err, plan_err}; use datafusion_expr::{Expr, UserDefinedLogicalNodeCore}; use educe::Educe; use sail_common_datafusion::datasource::{SinkInfo, SinkMode, SourceInfo, TableFormat}; @@ -17,8 +17,8 @@ use sail_common_datafusion::streaming::event::schema::is_flow_event_schema; use sail_common_datafusion::utils::items::ItemTaker; pub use crate::formats::console::writer::ConsoleSinkExec; -use crate::options::gen::ConsoleWriteOptions; use crate::options::ResolveOptions; +use crate::options::r#gen::ConsoleWriteOptions; /// Write data to stdout for testing purposes. #[derive(Debug)] diff --git a/crates/sail-data-source/src/formats/console/options.rs b/crates/sail-data-source/src/formats/console/options.rs index f485ef139..dc894e7aa 100644 --- a/crates/sail-data-source/src/formats/console/options.rs +++ b/crates/sail-data-source/src/formats/console/options.rs @@ -2,7 +2,7 @@ use datafusion::catalog::Session; use sail_common_datafusion::datasource::OptionLayer; use crate::error::DataSourceResult; -use crate::options::gen::{ConsoleWriteOptions, ConsoleWritePartialOptions}; +use crate::options::r#gen::{ConsoleWriteOptions, ConsoleWritePartialOptions}; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; impl ResolveOptions for ConsoleWriteOptions { diff --git a/crates/sail-data-source/src/formats/console/writer.rs b/crates/sail-data-source/src/formats/console/writer.rs index 8eba6be30..41e724262 100644 --- a/crates/sail-data-source/src/formats/console/writer.rs +++ b/crates/sail-data-source/src/formats/console/writer.rs @@ -8,7 +8,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; -use datafusion_common::{plan_err, Result}; +use datafusion_common::{Result, plan_err}; use futures::StreamExt; #[derive(Debug)] diff --git a/crates/sail-data-source/src/formats/csv/mod.rs b/crates/sail-data-source/src/formats/csv/mod.rs index 3a2eaed64..74ec10222 100644 --- a/crates/sail-data-source/src/formats/csv/mod.rs +++ b/crates/sail-data-source/src/formats/csv/mod.rs @@ -4,8 +4,8 @@ use datafusion_common::{DataFusionError, Result}; use sail_common_datafusion::datasource::OptionLayer; use crate::listing::source::{FormatFactory, ListingTableFormat}; -use crate::options::gen::{CsvReadOptions, CsvWriteOptions}; use crate::options::ResolveOptions; +use crate::options::r#gen::{CsvReadOptions, CsvWriteOptions}; // Some of the code in the `read` and `write` modules is adapted from the DataFusion `CsvFormat` implementation. // [CREDIT]: https://github.com/apache/datafusion/blob/53.1.0/datafusion/datasource-csv/src/file_format.rs diff --git a/crates/sail-data-source/src/formats/csv/options.rs b/crates/sail-data-source/src/formats/csv/options.rs index cc6f32b0b..8ddc13b13 100644 --- a/crates/sail-data-source/src/formats/csv/options.rs +++ b/crates/sail-data-source/src/formats/csv/options.rs @@ -6,7 +6,7 @@ use datafusion_datasource::file_compression_type::FileCompressionType; use sail_common_datafusion::datasource::OptionLayer; use crate::error::{DataSourceError, DataSourceResult}; -use crate::options::gen::{ +use crate::options::r#gen::{ CsvReadOptions, CsvReadPartialOptions, CsvWriteOptions, CsvWritePartialOptions, }; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; @@ -28,6 +28,7 @@ impl BuildPartialOptions for CsvOptions { multi_line: self.newlines_in_values, compression: Some(self.compression.to_string()), allow_truncated_rows: self.truncated_rows, + path_glob_filter: None, }) } } @@ -62,6 +63,7 @@ impl CsvReadOptions { multi_line, compression, allow_truncated_rows, + path_glob_filter: _, } = self; let null_regex = match (null_value.as_str(), null_regex.as_str()) { (nv, nr) if !nv.is_empty() && !nr.is_empty() => { @@ -235,8 +237,8 @@ mod tests { use datafusion::prelude::SessionContext; use datafusion_common::parsers::CompressionTypeVariant; - use crate::options::gen::{CsvReadOptions, CsvWriteOptions}; - use crate::options::{option_list, ResolveOptions}; + use crate::options::r#gen::{CsvReadOptions, CsvWriteOptions}; + use crate::options::{ResolveOptions, option_list}; #[test] fn test_resolve_csv_read_options() -> datafusion_common::Result<()> { diff --git a/crates/sail-data-source/src/formats/csv/read.rs b/crates/sail-data-source/src/formats/csv/read.rs index 91668dc4b..23a21e980 100644 --- a/crates/sail-data-source/src/formats/csv/read.rs +++ b/crates/sail-data-source/src/formats/csv/read.rs @@ -15,7 +15,7 @@ use object_store::ObjectStoreExt; use crate::listing::source::{ListingFileSample, ListingScanInput, ReadFormat}; use crate::listing::utils::infer_listing_compression; -use crate::options::gen::CsvReadOptions; +use crate::options::r#gen::CsvReadOptions; #[derive(Debug, Clone)] pub struct CsvReadFormat { @@ -73,7 +73,7 @@ impl ReadFormat for CsvReadFormat { .await .map_err(|err| { DataFusionError::Context( - format!("Error when processing CSV file {}", &object.location), + format!("Error when processing CSV file {}", object.location), Box::new(err), ) })?; @@ -130,4 +130,8 @@ impl ReadFormat for CsvReadFormat { Ok(config) } + + fn path_glob_filter(&self) -> Option<&str> { + self.options.path_glob_filter.as_deref() + } } diff --git a/crates/sail-data-source/src/formats/csv/write.rs b/crates/sail-data-source/src/formats/csv/write.rs index 3a0effb9c..1c1749638 100644 --- a/crates/sail-data-source/src/formats/csv/write.rs +++ b/crates/sail-data-source/src/formats/csv/write.rs @@ -8,7 +8,7 @@ use datafusion_common::{DataFusionError, Result}; use datafusion_datasource::file_format::FileFormat; use crate::listing::source::{ListingSinkInput, WriteFormat}; -use crate::options::gen::CsvWriteOptions; +use crate::options::r#gen::CsvWriteOptions; #[derive(Debug, Clone)] pub struct CsvWriteFormat { diff --git a/crates/sail-data-source/src/formats/json/mod.rs b/crates/sail-data-source/src/formats/json/mod.rs index 98231757b..764f33256 100644 --- a/crates/sail-data-source/src/formats/json/mod.rs +++ b/crates/sail-data-source/src/formats/json/mod.rs @@ -3,8 +3,8 @@ use datafusion_common::{DataFusionError, Result}; use sail_common_datafusion::datasource::OptionLayer; use crate::listing::source::{FormatFactory, ListingTableFormat}; -use crate::options::gen::{JsonReadOptions, JsonWriteOptions}; use crate::options::ResolveOptions; +use crate::options::r#gen::{JsonReadOptions, JsonWriteOptions}; // Some of the code in the `read` and `write` modules is adapted from the DataFusion `JsonFormat` implementation. // [CREDIT]: https://github.com/apache/datafusion/blob/53.1.0/datafusion/datasource-json/src/file_format.rs diff --git a/crates/sail-data-source/src/formats/json/options.rs b/crates/sail-data-source/src/formats/json/options.rs index 3f8d6b65c..91aec3613 100644 --- a/crates/sail-data-source/src/formats/json/options.rs +++ b/crates/sail-data-source/src/formats/json/options.rs @@ -6,7 +6,7 @@ use datafusion_datasource::file_compression_type::FileCompressionType; use sail_common_datafusion::datasource::OptionLayer; use crate::error::{DataSourceError, DataSourceResult}; -use crate::options::gen::{ +use crate::options::r#gen::{ JsonReadOptions, JsonReadPartialOptions, JsonWriteOptions, JsonWritePartialOptions, }; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; @@ -16,6 +16,7 @@ impl BuildPartialOptions for JsonOptions { Ok(JsonReadPartialOptions { schema_infer_max_records: self.schema_infer_max_rec, compression: Some(self.compression.to_string()), + path_glob_filter: None, }) } } @@ -33,6 +34,7 @@ impl JsonReadOptions { let JsonReadOptions { schema_infer_max_records, compression, + path_glob_filter: _, } = self; let compression = FileCompressionType::from_str(&compression) .map_err(|e| DataSourceError::InvalidOption { @@ -93,8 +95,8 @@ mod tests { use datafusion::prelude::SessionContext; use datafusion_common::parsers::CompressionTypeVariant; - use crate::options::gen::{JsonReadOptions, JsonWriteOptions}; - use crate::options::{option_list, ResolveOptions}; + use crate::options::r#gen::{JsonReadOptions, JsonWriteOptions}; + use crate::options::{ResolveOptions, option_list}; #[test] fn test_resolve_json_read_options() -> datafusion_common::Result<()> { diff --git a/crates/sail-data-source/src/formats/json/read.rs b/crates/sail-data-source/src/formats/json/read.rs index 39056ed34..916638943 100644 --- a/crates/sail-data-source/src/formats/json/read.rs +++ b/crates/sail-data-source/src/formats/json/read.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bytes::Buf; use datafusion::arrow::datatypes::{Schema, SchemaRef}; -use datafusion::arrow::json::reader::{infer_json_schema_from_iterator, ValueIter}; +use datafusion::arrow::json::reader::{ValueIter, infer_json_schema_from_iterator}; use datafusion::catalog::Session; use datafusion::datasource::physical_plan::JsonSource; use datafusion_common::parsers::CompressionTypeVariant; @@ -15,7 +15,7 @@ use object_store::{GetResultPayload, ObjectStoreExt}; use crate::listing::source::{ListingFileSample, ListingScanInput, ReadFormat}; use crate::listing::utils::infer_listing_compression; -use crate::options::gen::JsonReadOptions; +use crate::options::r#gen::JsonReadOptions; #[derive(Debug, Clone)] pub struct JsonReadFormat { @@ -116,6 +116,10 @@ impl ReadFormat for JsonReadFormat { Ok(config) } + + fn path_glob_filter(&self) -> Option<&str> { + self.options.path_glob_filter.as_deref() + } } /// A tuple of (Schema, records_consumed) where records_consumed is the number of records that were diff --git a/crates/sail-data-source/src/formats/json/write.rs b/crates/sail-data-source/src/formats/json/write.rs index c85956313..2cf597e89 100644 --- a/crates/sail-data-source/src/formats/json/write.rs +++ b/crates/sail-data-source/src/formats/json/write.rs @@ -8,7 +8,7 @@ use datafusion_common::{DataFusionError, Result}; use datafusion_datasource::file_format::FileFormat; use crate::listing::source::{ListingSinkInput, WriteFormat}; -use crate::options::gen::JsonWriteOptions; +use crate::options::r#gen::JsonWriteOptions; #[derive(Debug, Clone)] pub struct JsonWriteFormat { diff --git a/crates/sail-data-source/src/formats/noop/mod.rs b/crates/sail-data-source/src/formats/noop/mod.rs index 8c5f5d615..90f74c31f 100644 --- a/crates/sail-data-source/src/formats/noop/mod.rs +++ b/crates/sail-data-source/src/formats/noop/mod.rs @@ -6,13 +6,13 @@ use datafusion::execution::SessionState; use datafusion::logical_expr::{Extension, LogicalPlan, TableSource, UserDefinedLogicalNode}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; -use datafusion_common::{internal_err, not_impl_err, DFSchema, DFSchemaRef, Result}; +use datafusion_common::{DFSchema, DFSchemaRef, Result, internal_err, not_impl_err}; use datafusion_expr::{Expr, UserDefinedLogicalNodeCore}; use educe::Educe; use sail_common_datafusion::datasource::{SinkInfo, SourceInfo, TableFormat}; use sail_common_datafusion::utils::items::ItemTaker; -use crate::formats::noop::writer::NoopSinkExec; +pub use crate::formats::noop::writer::NoopSinkExec; mod writer; @@ -38,7 +38,6 @@ impl TableFormat for NoopTableFormat { input, partition_by, bucket_by, - sort_order: _, .. } = info; diff --git a/crates/sail-data-source/src/formats/noop/writer.rs b/crates/sail-data-source/src/formats/noop/writer.rs index a5f36a4ec..4460c038d 100644 --- a/crates/sail-data-source/src/formats/noop/writer.rs +++ b/crates/sail-data-source/src/formats/noop/writer.rs @@ -6,7 +6,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; -use datafusion_common::{plan_err, Result}; +use datafusion_common::{Result, plan_err}; use futures::{StreamExt, TryStreamExt}; #[derive(Debug)] @@ -27,6 +27,10 @@ impl NoopSinkExec { )); Self { input, properties } } + + pub fn input(&self) -> &Arc { + &self.input + } } impl DisplayAs for NoopSinkExec { diff --git a/crates/sail-data-source/src/formats/parquet/mod.rs b/crates/sail-data-source/src/formats/parquet/mod.rs index 0abfe35c2..3da695575 100644 --- a/crates/sail-data-source/src/formats/parquet/mod.rs +++ b/crates/sail-data-source/src/formats/parquet/mod.rs @@ -3,8 +3,8 @@ use datafusion_common::{DataFusionError, Result}; use sail_common_datafusion::datasource::OptionLayer; use crate::listing::source::{FormatFactory, ListingTableFormat}; -use crate::options::gen::{ParquetReadOptions, ParquetWriteOptions}; use crate::options::ResolveOptions; +use crate::options::r#gen::{ParquetReadOptions, ParquetWriteOptions}; // Some of the code in the `read` and `write` modules is adapted from the DataFusion `ParquetFormat` implementation. // [CREDIT]: https://github.com/apache/datafusion/blob/53.1.0/datafusion/datasource-parquet/src/file_format.rs diff --git a/crates/sail-data-source/src/formats/parquet/options.rs b/crates/sail-data-source/src/formats/parquet/options.rs index 057a36266..c4f2830ec 100644 --- a/crates/sail-data-source/src/formats/parquet/options.rs +++ b/crates/sail-data-source/src/formats/parquet/options.rs @@ -7,7 +7,7 @@ use datafusion_common::parquet_config::DFParquetWriterVersion; use sail_common_datafusion::datasource::OptionLayer; use crate::error::{DataSourceError, DataSourceResult}; -use crate::options::gen::{ +use crate::options::r#gen::{ ParquetReadOptions, ParquetReadPartialOptions, ParquetWriteOptions, ParquetWritePartialOptions, }; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; @@ -27,7 +27,6 @@ fn check_parquet_level_is_none(codec: &str, level: &Option) -> DataSourceRe impl BuildPartialOptions for TableParquetOptions { fn build_partial_options(self) -> DataSourceResult { Ok(ParquetReadPartialOptions { - extension: None, enable_page_index: Some(self.global.enable_page_index), pruning: Some(self.global.pruning), skip_metadata: Some(self.global.skip_metadata), @@ -40,6 +39,7 @@ impl BuildPartialOptions for TableParquetOptions { coerce_int96: self.global.coerce_int96.map(Some), bloom_filter_on_read: Some(self.global.bloom_filter_on_read), max_predicate_cache_size: Some(self.global.max_predicate_cache_size), + path_glob_filter: None, }) } } @@ -47,7 +47,6 @@ impl BuildPartialOptions for TableParquetOptions { impl ParquetReadOptions { pub fn into_table_options(self) -> TableParquetOptions { let ParquetReadOptions { - extension: _, enable_page_index, pruning, skip_metadata, @@ -59,6 +58,7 @@ impl ParquetReadOptions { coerce_int96, bloom_filter_on_read, max_predicate_cache_size, + path_glob_filter: _, } = self; let global = ParquetOptions { enable_page_index, @@ -312,8 +312,8 @@ mod tests { use datafusion::prelude::SessionContext; use datafusion_common::parquet_config::DFParquetWriterVersion; - use crate::options::gen::{ParquetReadOptions, ParquetWriteOptions}; - use crate::options::{option_list, ResolveOptions}; + use crate::options::r#gen::{ParquetReadOptions, ParquetWriteOptions}; + use crate::options::{ResolveOptions, option_list}; #[test] fn test_resolve_parquet_read_options() -> datafusion_common::Result<()> { @@ -367,36 +367,6 @@ mod tests { Ok(()) } - #[test] - fn test_resolve_parquet_read_extension() -> datafusion_common::Result<()> { - let ctx = SessionContext::default(); - let state = ctx.state(); - - // default: no option layer → ".parquet" - let opts = ParquetReadOptions::resolve(&state, vec![option_list(&[])]) - .map_err(datafusion_common::DataFusionError::from)?; - assert_eq!(opts.extension, ".parquet"); - - // snake_case override - let opts = - ParquetReadOptions::resolve(&state, vec![option_list(&[("extension", ".hive")])]) - .map_err(datafusion_common::DataFusionError::from)?; - assert_eq!(opts.extension, ".hive"); - - // camelCase alias - let opts = - ParquetReadOptions::resolve(&state, vec![option_list(&[("fileExtension", ".pq")])]) - .map_err(datafusion_common::DataFusionError::from)?; - assert_eq!(opts.extension, ".pq"); - - // empty string disables filtering - let opts = ParquetReadOptions::resolve(&state, vec![option_list(&[("extension", "")])]) - .map_err(datafusion_common::DataFusionError::from)?; - assert_eq!(opts.extension, ""); - - Ok(()) - } - #[test] fn test_resolve_parquet_read_options_with_global_default() -> datafusion_common::Result<()> { let ctx = SessionContext::default(); diff --git a/crates/sail-data-source/src/formats/parquet/read.rs b/crates/sail-data-source/src/formats/parquet/read.rs index 31e947f64..f71cba357 100644 --- a/crates/sail-data-source/src/formats/parquet/read.rs +++ b/crates/sail-data-source/src/formats/parquet/read.rs @@ -1,22 +1,22 @@ use std::sync::Arc; -use datafusion::arrow::datatypes::{Fields, Schema, SchemaRef, TimeUnit}; +use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef, TimeUnit}; use datafusion::catalog::Session; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::datasource::physical_plan::parquet::CachedParquetFileReaderFactory; use datafusion::datasource::physical_plan::parquet::metadata::{ - ordering_from_parquet_metadata, DFParquetMetadata, + DFParquetMetadata, ordering_from_parquet_metadata, }; -use datafusion::datasource::physical_plan::parquet::CachedParquetFileReaderFactory; -use datafusion::datasource::physical_plan::ParquetSource; use datafusion_common::config::TableParquetOptions; use datafusion_common::parsers::CompressionTypeVariant; -use datafusion_common::{DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, plan_err}; use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; use futures::{StreamExt, TryStreamExt}; use object_store::{ObjectMeta, ObjectStore}; use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapterFactory; use crate::listing::source::{ListingFileMeta, ListingFileSample, ListingScanInput, ReadFormat}; -use crate::options::gen::ParquetReadOptions; +use crate::options::r#gen::ParquetReadOptions; #[derive(Debug, Clone)] pub struct ParquetReadFormat { @@ -166,6 +166,155 @@ impl ReadFormat for ParquetReadFormat { Ok(config) } + + fn requires_explicit_schema_validation(&self) -> bool { + true + } + + fn validate_explicit_schema(&self, schema: &Schema, physical: &Schema) -> Result<()> { + validate_parquet_schema(schema, physical) + } + + fn path_glob_filter(&self) -> Option<&str> { + self.options.path_glob_filter.as_deref() + } +} + +fn validate_parquet_schema(schema: &Schema, physical: &Schema) -> Result<()> { + for field in schema.fields() { + if let Ok(physical_field) = physical.field_with_name(field.name()) { + validate_parquet_field(field.name(), field, physical_field)?; + } + } + Ok(()) +} + +fn validate_parquet_field(path: &str, requested: &Field, physical: &Field) -> Result<()> { + match (physical.data_type(), requested.data_type()) { + (DataType::Struct(physical_fields), DataType::Struct(requested_fields)) => { + for requested_field in requested_fields { + if let Some(physical_field) = physical_fields + .iter() + .find(|field| field.name() == requested_field.name()) + { + validate_parquet_field( + &format!("{path}.{}", requested_field.name()), + requested_field, + physical_field, + )?; + } + } + Ok(()) + } + (DataType::List(physical_element), DataType::List(requested_element)) + | (DataType::LargeList(physical_element), DataType::LargeList(requested_element)) => { + validate_parquet_field(&format!("{path}[]"), requested_element, physical_element) + } + ( + DataType::FixedSizeList(physical_element, physical_length), + DataType::FixedSizeList(requested_element, requested_length), + ) if physical_length == requested_length => { + validate_parquet_field(&format!("{path}[]"), requested_element, physical_element) + } + (DataType::Map(physical_entries, _), DataType::Map(requested_entries, _)) => { + validate_parquet_field(path, requested_entries, physical_entries) + } + (physical_type, requested_type) + if parquet_primitive_type_compatible(physical_type, requested_type) => + { + Ok(()) + } + (physical_type, requested_type) => plan_err!( + "[FAILED_READ_FILE.PARQUET_COLUMN_DATA_TYPE_MISMATCH] Data type mismatches when reading \ + Parquet column `{path}`. Expected {requested_type}, actual Parquet type {physical_type}." + ), + } +} + +fn parquet_primitive_type_compatible(physical: &DataType, requested: &DataType) -> bool { + if physical == requested { + return true; + } + + match (physical, requested) { + // Spark's Parquet readers support these lossless numeric widenings. + (DataType::Int32, DataType::Int64 | DataType::Float64) + | (DataType::Float32, DataType::Float64) + | (DataType::UInt8, DataType::Int16) + | (DataType::UInt16, DataType::Int32) + | (DataType::UInt32, DataType::Int64) => true, + (DataType::UInt64, DataType::Decimal128(20, 0)) => true, + ( + DataType::Int8 | DataType::Int16 | DataType::Int32, + DataType::Decimal128(precision, scale), + ) => decimal_integer_digits(*precision, *scale) >= 10, + (DataType::Int64, DataType::Decimal128(precision, scale)) => { + decimal_integer_digits(*precision, *scale) >= 20 + } + ( + DataType::Decimal128(physical_precision, physical_scale), + DataType::Decimal128(requested_precision, requested_scale), + ) + | ( + DataType::Decimal256(physical_precision, physical_scale), + DataType::Decimal256(requested_precision, requested_scale), + ) => decimal_type_compatible( + *physical_precision, + *physical_scale, + *requested_precision, + *requested_scale, + ), + ( + DataType::Decimal128(physical_precision, physical_scale), + DataType::Decimal256(requested_precision, requested_scale), + ) => decimal_type_compatible( + *physical_precision, + *physical_scale, + *requested_precision, + *requested_scale, + ), + // Parquet timestamps may differ in storage precision, but Spark keeps the + // local-time-zone and no-time-zone families distinct. + (DataType::Timestamp(_, physical_timezone), DataType::Timestamp(_, requested_timezone)) => { + physical_timezone.is_some() == requested_timezone.is_some() + } + (DataType::Date32, DataType::Timestamp(_, None)) => true, + (physical, requested) if is_string_type(physical) && is_string_type(requested) => true, + (physical, requested) if is_binary_type(physical) && is_binary_type(requested) => true, + _ => false, + } +} + +fn decimal_integer_digits(precision: u8, scale: i8) -> i16 { + i16::from(precision) - i16::from(scale) +} + +fn decimal_type_compatible( + physical_precision: u8, + physical_scale: i8, + requested_precision: u8, + requested_scale: i8, +) -> bool { + let scale_increase = i16::from(requested_scale) - i16::from(physical_scale); + let precision_increase = i16::from(requested_precision) - i16::from(physical_precision); + scale_increase >= 0 && precision_increase >= scale_increase +} + +fn is_string_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView + | DataType::FixedSizeBinary(_) + ) } /// Clears all metadata (Schema level and field level) for a schema. diff --git a/crates/sail-data-source/src/formats/parquet/write.rs b/crates/sail-data-source/src/formats/parquet/write.rs index b7d94a982..b86ea6b8f 100644 --- a/crates/sail-data-source/src/formats/parquet/write.rs +++ b/crates/sail-data-source/src/formats/parquet/write.rs @@ -10,7 +10,7 @@ use datafusion_datasource::file_compression_type::FileCompressionType; use datafusion_datasource::file_format::FileFormat; use crate::listing::source::{ListingSinkInput, WriteFormat}; -use crate::options::gen::ParquetWriteOptions; +use crate::options::r#gen::ParquetWriteOptions; use crate::utils::split_parquet_compression_string; #[derive(Debug, Clone)] diff --git a/crates/sail-data-source/src/formats/python/arrow_utils.rs b/crates/sail-data-source/src/formats/python/arrow_utils.rs index d87cca6ee..21a6d964d 100644 --- a/crates/sail-data-source/src/formats/python/arrow_utils.rs +++ b/crates/sail-data-source/src/formats/python/arrow_utils.rs @@ -19,9 +19,9 @@ use std::sync::Arc; use arrow::array::{ - ArrayRef, BooleanArray, Date32Array, Decimal128Array, Float32Array, Float64Array, Int16Array, - Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeStringArray, NullArray, RecordBatch, - StringArray, TimestampMicrosecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array, + ArrayRef, BooleanArray, Date32Array, Decimal128Array, Float32Array, Float64Array, Int8Array, + Int16Array, Int32Array, Int64Array, LargeBinaryArray, LargeStringArray, NullArray, RecordBatch, + StringArray, TimestampMicrosecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, }; use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use arrow_schema::SchemaRef; @@ -151,7 +151,7 @@ pub fn record_batch_to_py_rows(py: Python<'_>, batch: &RecordBatch) -> Result {{ + ($arr:expr_2021, $row_idx:expr_2021, $py:expr_2021) => {{ use pyo3::IntoPyObject; $arr.value($row_idx) .into_pyobject($py) @@ -510,7 +510,7 @@ pub fn convert_rows_to_batch(schema: &SchemaRef, pickled_rows: &[Vec]) -> Re /// /// Generates the common pattern of extracting values from rows and building an array. macro_rules! build_primitive_array { - ($rows:expr, $col_idx:expr, $rust_ty:ty, $array_ty:ty) => {{ + ($rows:expr_2021, $col_idx:expr_2021, $rust_ty:ty, $array_ty:ty) => {{ let values: Vec> = $rows .iter() .map(|row| extract_value(row, $col_idx)) @@ -528,7 +528,7 @@ fn build_array_from_rows( ) -> Result { /// Helper macro for string-like types. macro_rules! build_string_array { - ($rows:expr, $col_idx:expr, $array_ty:ty) => {{ + ($rows:expr_2021, $col_idx:expr_2021, $array_ty:ty) => {{ let values: Vec> = $rows .iter() .map(|row| extract_value(row, $col_idx)) @@ -541,7 +541,7 @@ fn build_array_from_rows( /// Helper macro for binary-like types. macro_rules! build_binary_array { - ($rows:expr, $col_idx:expr, $array_ty:ty) => {{ + ($rows:expr_2021, $col_idx:expr_2021, $array_ty:ty) => {{ let values: Vec>> = $rows .iter() .map(|row| extract_value(row, $col_idx)) diff --git a/crates/sail-data-source/src/formats/python/commit_exec.rs b/crates/sail-data-source/src/formats/python/commit_exec.rs index 589c4f124..0fa2a63b7 100644 --- a/crates/sail-data-source/src/formats/python/commit_exec.rs +++ b/crates/sail-data-source/src/formats/python/commit_exec.rs @@ -15,7 +15,7 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, }; -use datafusion_common::{exec_err, internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, exec_err, internal_err}; use futures::StreamExt; use super::executor::{InProcessExecutor, PythonExecutor}; @@ -162,10 +162,9 @@ impl ExecutionPlan for PythonDataSourceWriteCommitExec { &mut commit_messages, &mut seen_partitions, &mut first_error, - ) { - if first_error.is_none() { - first_error = Some(e.to_string()); - } + ) && first_error.is_none() + { + first_error = Some(e.to_string()); } } @@ -352,9 +351,11 @@ mod tests { let exec = Arc::new(PythonDataSourceWriteCommitExec::new(input1, vec![], 2)); let new_exec = exec.clone().with_new_children(vec![input2]).unwrap(); - assert!(new_exec - .downcast_ref::() - .is_some()); + assert!( + new_exec + .downcast_ref::() + .is_some() + ); } #[test] diff --git a/crates/sail-data-source/src/formats/python/datasource.rs b/crates/sail-data-source/src/formats/python/datasource.rs index f19d2864c..8ec6fdd53 100644 --- a/crates/sail-data-source/src/formats/python/datasource.rs +++ b/crates/sail-data-source/src/formats/python/datasource.rs @@ -11,7 +11,7 @@ use pyo3::prelude::*; use pyo3::types::PyBytes; use super::arrow_utils::py_schema_to_rust; -use super::error::{import_cloudpickle, PythonDataSourceContext, PythonDataSourceError}; +use super::error::{PythonDataSourceContext, PythonDataSourceError, import_cloudpickle}; /// Represents a Python data source. /// diff --git a/crates/sail-data-source/src/formats/python/discovery.rs b/crates/sail-data-source/src/formats/python/discovery.rs index 1ecfb7d83..02c8b5f96 100644 --- a/crates/sail-data-source/src/formats/python/discovery.rs +++ b/crates/sail-data-source/src/formats/python/discovery.rs @@ -6,9 +6,9 @@ //! - Data source validation for security use std::ffi::CString; -use dashmap::mapref::entry::Entry; use dashmap::DashMap; -use datafusion_common::{exec_err, Result}; +use dashmap::mapref::entry::Entry; +use datafusion_common::{Result, exec_err}; use once_cell::sync::Lazy; use pyo3::prelude::PyModule; use pyo3::sync::PyOnceLock; diff --git a/crates/sail-data-source/src/formats/python/exec.rs b/crates/sail-data-source/src/formats/python/exec.rs index b54432ba5..9bd3754e2 100644 --- a/crates/sail-data-source/src/formats/python/exec.rs +++ b/crates/sail-data-source/src/formats/python/exec.rs @@ -35,7 +35,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use datafusion_common::{exec_err, internal_err, Result}; +use datafusion_common::{Result, exec_err, internal_err}; use super::executor::InputPartition; diff --git a/crates/sail-data-source/src/formats/python/executor.rs b/crates/sail-data-source/src/formats/python/executor.rs index afc427dad..cb33fd20a 100644 --- a/crates/sail-data-source/src/formats/python/executor.rs +++ b/crates/sail-data-source/src/formats/python/executor.rs @@ -13,8 +13,8 @@ use futures::stream::BoxStream; use pyo3::prelude::*; use pyo3::types::PyAnyMethods; -use super::error::{import_cloudpickle, PythonDataSourceContext}; -use super::filter::{filters_to_python, PythonFilter}; +use super::error::{PythonDataSourceContext, import_cloudpickle}; +use super::filter::{PythonFilter, filters_to_python}; /// Default capacity for the write channel, matching `python.data_source_write_channel_capacity` config. const DEFAULT_WRITE_CHANNEL_CAPACITY: usize = 8; @@ -673,14 +673,17 @@ impl PythonExecutor for InProcessExecutor { }; // Call writer.abort(messages) - log errors but don't propagate - if let Err(e) = writer.call_method1("abort", (messages_list,)) { - log::warn!("[{}::abort] Abort call failed: {}", writer_name, e); - } else { - log::debug!( - "[{}::abort] Aborted {} partitions", - writer_name, - commit_messages.len() - ); + match writer.call_method1("abort", (messages_list,)) { + Err(e) => { + log::warn!("[{}::abort] Abort call failed: {}", writer_name, e); + } + _ => { + log::debug!( + "[{}::abort] Aborted {} partitions", + writer_name, + commit_messages.len() + ); + } } Ok(()) @@ -740,49 +743,46 @@ impl RecordBatchIterator { loop { // If we have an active batch, try to get the next item - if let Some(batch) = &slf.current_batch { - if slf.current_row < batch.num_rows() { - // Compute result and next row index without mutating slf while borrowing batch - let (result, next_row) = if slf.is_arrow { - // For arrow, return one batch at a time (DataSourceArrowWriter.write(iterator)) - let r = super::arrow_utils::rust_record_batch_to_py(py, batch).map_err( - |e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()), - )?; - (r, batch.num_rows()) - } else { - // For row, return one row at a time - let row_idx = slf.current_row; - let factory = slf - .row_factory - .as_ref() - .ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err( - "row_factory is None for row-based writer", - ) - })? - .bind(py); - let mut row_values = Vec::with_capacity(batch.num_columns()); - for col_idx in 0..batch.num_columns() { - let column = batch.column(col_idx); - let value = - super::arrow_utils::extract_python_value(py, column, row_idx) - .map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(e.to_string()) - })?; - row_values.push(value); - } - let args = pyo3::types::PyTuple::new(py, row_values).map_err(|e| { + if let Some(batch) = &slf.current_batch + && slf.current_row < batch.num_rows() + { + // Compute result and next row index without mutating slf while borrowing batch + let (result, next_row) = if slf.is_arrow { + // For arrow, return one batch at a time (DataSourceArrowWriter.write(iterator)) + let r = super::arrow_utils::rust_record_batch_to_py(py, batch) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + (r, batch.num_rows()) + } else { + // For row, return one row at a time + let row_idx = slf.current_row; + let factory = slf + .row_factory + .as_ref() + .ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err( + "row_factory is None for row-based writer", + ) + })? + .bind(py); + let mut row_values = Vec::with_capacity(batch.num_columns()); + for col_idx in 0..batch.num_columns() { + let column = batch.column(col_idx); + let value = super::arrow_utils::extract_python_value(py, column, row_idx) + .map_err(|e| { pyo3::exceptions::PyRuntimeError::new_err(e.to_string()) })?; - let r = factory - .call1(args) - .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))? - .unbind(); - (r, row_idx + 1) - }; - slf.current_row = next_row; - return Ok(Some(result)); - } + row_values.push(value); + } + let args = pyo3::types::PyTuple::new(py, row_values) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + let r = factory + .call1(args) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))? + .unbind(); + (r, row_idx + 1) + }; + slf.current_row = next_row; + return Ok(Some(result)); } // No active batch or batch exhausted, get the next one from the channel diff --git a/crates/sail-data-source/src/formats/python/filter.rs b/crates/sail-data-source/src/formats/python/filter.rs index 931211fba..e356c8a20 100644 --- a/crates/sail-data-source/src/formats/python/filter.rs +++ b/crates/sail-data-source/src/formats/python/filter.rs @@ -11,9 +11,9 @@ //! - String patterns: StartsWith, EndsWith, Contains use datafusion::logical_expr::{Expr, Operator}; use datafusion_common::ScalarValue; +use pyo3::IntoPyObject; use pyo3::prelude::*; use pyo3::types::{PyAnyMethods, PyList, PyTuple}; -use pyo3::IntoPyObject; use serde::{Deserialize, Serialize}; /// Represents a filter that can be pushed to Python. diff --git a/crates/sail-data-source/src/formats/python/mod.rs b/crates/sail-data-source/src/formats/python/mod.rs index 0299f09ce..27d2091d5 100644 --- a/crates/sail-data-source/src/formats/python/mod.rs +++ b/crates/sail-data-source/src/formats/python/mod.rs @@ -39,14 +39,14 @@ mod write_exec; pub use commit_exec::PythonDataSourceWriteCommitExec; pub use datasource::PythonDataSource; pub use discovery::{ - discover_data_sources, validate_datasource_class, DataSourceEntry, PythonDataSourceRegistry, - DATA_SOURCE_REGISTRY, + DATA_SOURCE_REGISTRY, DataSourceEntry, PythonDataSourceRegistry, discover_data_sources, + validate_datasource_class, }; pub use error::PythonDataSourceError; pub use exec::PythonDataSourceExec; pub use executor::{InProcessExecutor, InputPartition, PythonExecutor}; -pub use filter::{exprs_to_python_filters, ColumnPath, FilterValue, PythonFilter}; -pub use stream::{PythonDataSourceStream, RowBatchCollector, DEFAULT_BATCH_SIZE}; +pub use filter::{ColumnPath, FilterValue, PythonFilter, exprs_to_python_filters}; +pub use stream::{DEFAULT_BATCH_SIZE, PythonDataSourceStream, RowBatchCollector}; pub use table_format::{PythonPhysicalPlanner, PythonTableFormat}; pub use table_provider::PythonTableProvider; pub use write_exec::PythonDataSourceWriteExec; diff --git a/crates/sail-data-source/src/formats/python/stream.rs b/crates/sail-data-source/src/formats/python/stream.rs index 4592a5c61..74f48a053 100644 --- a/crates/sail-data-source/src/formats/python/stream.rs +++ b/crates/sail-data-source/src/formats/python/stream.rs @@ -63,7 +63,12 @@ impl PythonExecutionMetrics { log::trace!( "[PythonDataSource:partition={}] GIL metrics: wait={}ms, hold={}ms, acquisitions={}, batches={}, rows={}", - partition_id, wait_ms, hold_ms, acquisitions, batches, rows + partition_id, + wait_ms, + hold_ms, + acquisitions, + batches, + rows ); // Log warning if GIL contention is high @@ -71,7 +76,9 @@ impl PythonExecutionMetrics { log::warn!( "[PythonDataSource:partition={}] High GIL contention detected: wait={}ms > hold={}ms. \ Consider using GIL-releasing libraries (connector-x, DuckDB) or subprocess isolation.", - partition_id, wait_ms, hold_ms + partition_id, + wait_ms, + hold_ms ); } } @@ -205,7 +212,9 @@ impl PythonDataSourceStream { log::warn!( "[PythonDataSource:partition={}] Slow read detected: {}ms (threshold: {}ms). \ Consider using faster I/O or subprocess isolation.", - partition_id, read_elapsed_ms, slow_read_warn_ms + partition_id, + read_elapsed_ms, + slow_read_warn_ms ); } @@ -246,16 +255,16 @@ impl PythonDataSourceStream { row_batcher.add_row(row_bytes); // Flush if batch is ready - if row_batcher.is_ready() { - if let Some(batch) = row_batcher.flush()? { - metrics - .rows_processed - .fetch_add(batch.num_rows() as u64, Ordering::Relaxed); - metrics.batches_processed.fetch_add(1, Ordering::Relaxed); - - if tx.blocking_send(Ok(batch)).is_err() { - break; - } + if row_batcher.is_ready() + && let Some(batch) = row_batcher.flush()? + { + metrics + .rows_processed + .fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + metrics.batches_processed.fetch_add(1, Ordering::Relaxed); + + if tx.blocking_send(Ok(batch)).is_err() { + break; } } } diff --git a/crates/sail-data-source/src/formats/python/table_format.rs b/crates/sail-data-source/src/formats/python/table_format.rs index 1cd235072..abda76458 100644 --- a/crates/sail-data-source/src/formats/python/table_format.rs +++ b/crates/sail-data-source/src/formats/python/table_format.rs @@ -12,7 +12,7 @@ use datafusion::execution::SessionState; use datafusion::logical_expr::{Extension, LogicalPlan, TableSource, UserDefinedLogicalNode}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; -use datafusion_common::{internal_err, DFSchema, DFSchemaRef, Result}; +use datafusion_common::{DFSchema, DFSchemaRef, Result, internal_err}; use datafusion_expr::{Expr, UserDefinedLogicalNodeCore}; use educe::Educe; use sail_common_datafusion::datasource::{ @@ -145,11 +145,16 @@ impl PythonTableFormat { .call_method1("loads", (class_bytes,)) .map_err(py_err)?; - // Create options dict + // Create Spark-compatible case-insensitive options let py_options = PyDict::new(py); for (k, v) in &options { py_options.set_item(k, v).map_err(py_err)?; } + let py_options = py + .import("pyspark.sql.datasource") + .and_then(|module| module.getattr("CaseInsensitiveDict")) + .and_then(|class| class.call1((py_options,))) + .map_err(py_err)?; // Instantiate the datasource with options let ds_instance = ds_class.call1((py_options,)).map_err(py_err)?; diff --git a/crates/sail-data-source/src/formats/python/write_exec.rs b/crates/sail-data-source/src/formats/python/write_exec.rs index 6dd26af43..ed3c9c56b 100644 --- a/crates/sail-data-source/src/formats/python/write_exec.rs +++ b/crates/sail-data-source/src/formats/python/write_exec.rs @@ -20,7 +20,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use datafusion_common::{internal_err, Result}; +use datafusion_common::{Result, internal_err}; use futures::StreamExt; use super::executor::{InProcessExecutor, PythonExecutor}; @@ -257,9 +257,11 @@ mod tests { let new_exec = exec.clone().with_new_children(vec![input2]).unwrap(); - assert!(new_exec - .downcast_ref::() - .is_some()); + assert!( + new_exec + .downcast_ref::() + .is_some() + ); } #[test] @@ -273,17 +275,18 @@ mod tests { let exec = Arc::new(PythonDataSourceWriteExec::new(input, vec![], true)); assert!(exec.clone().with_new_children(vec![]).is_err()); - assert!(exec - .clone() - .with_new_children(vec![ - Arc::new(datafusion::physical_plan::empty::EmptyExec::new( - schema.clone() - )), - Arc::new(datafusion::physical_plan::empty::EmptyExec::new( - schema.clone() - )) - ]) - .is_err()); + assert!( + exec.clone() + .with_new_children(vec![ + Arc::new(datafusion::physical_plan::empty::EmptyExec::new( + schema.clone() + )), + Arc::new(datafusion::physical_plan::empty::EmptyExec::new( + schema.clone() + )) + ]) + .is_err() + ); } #[test] diff --git a/crates/sail-data-source/src/formats/rate/mod.rs b/crates/sail-data-source/src/formats/rate/mod.rs index ab880c6a2..77926d16c 100644 --- a/crates/sail-data-source/src/formats/rate/mod.rs +++ b/crates/sail-data-source/src/formats/rate/mod.rs @@ -9,14 +9,14 @@ use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datafusion::catalog::Session; use datafusion::datasource::provider_as_source; use datafusion::logical_expr::{LogicalPlan, TableSource}; -use datafusion_common::{plan_err, Result}; +use datafusion_common::{Result, plan_err}; use sail_common_datafusion::datasource::{SinkInfo, SourceInfo, TableFormat}; use sail_common_datafusion::streaming::source::StreamSourceTableProvider; pub use crate::formats::rate::reader::RateSourceExec; use crate::formats::rate::reader::RateStreamSource; -use crate::options::gen::RateReadOptions; use crate::options::ResolveOptions; +use crate::options::r#gen::RateReadOptions; /// Generate record batches at a fixed rate for testing purposes. /// The record batches contain two columns, a timestamp and an integer value. diff --git a/crates/sail-data-source/src/formats/rate/options.rs b/crates/sail-data-source/src/formats/rate/options.rs index 3f2b5c66a..a4ed07ce4 100644 --- a/crates/sail-data-source/src/formats/rate/options.rs +++ b/crates/sail-data-source/src/formats/rate/options.rs @@ -2,7 +2,7 @@ use datafusion::catalog::Session; use sail_common_datafusion::datasource::OptionLayer; use crate::error::DataSourceResult; -use crate::options::gen::{RateReadOptions, RateReadPartialOptions}; +use crate::options::r#gen::{RateReadOptions, RateReadPartialOptions}; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; impl ResolveOptions for RateReadOptions { diff --git a/crates/sail-data-source/src/formats/rate/reader.rs b/crates/sail-data-source/src/formats/rate/reader.rs index a8af34cfd..927e5fa08 100644 --- a/crates/sail-data-source/src/formats/rate/reader.rs +++ b/crates/sail-data-source/src/formats/rate/reader.rs @@ -11,15 +11,15 @@ use datafusion::logical_expr::Expr; use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; -use datafusion_common::{arrow_datafusion_err, plan_err, Result}; +use datafusion_common::{Result, arrow_datafusion_err, plan_err}; use futures::{Stream, StreamExt}; +use sail_common_datafusion::streaming::event::FlowEvent; use sail_common_datafusion::streaming::event::encoding::EncodedFlowEventStream; use sail_common_datafusion::streaming::event::schema::to_flow_event_schema; use sail_common_datafusion::streaming::event::stream::FlowEventStreamAdapter; -use sail_common_datafusion::streaming::event::FlowEvent; use sail_common_datafusion::streaming::source::StreamSource; -use crate::options::gen::RateReadOptions; +use crate::options::r#gen::RateReadOptions; #[derive(Debug, Clone)] pub struct RateStreamSource { diff --git a/crates/sail-data-source/src/formats/socket/mod.rs b/crates/sail-data-source/src/formats/socket/mod.rs index 3ae69d3d8..b84236abb 100644 --- a/crates/sail-data-source/src/formats/socket/mod.rs +++ b/crates/sail-data-source/src/formats/socket/mod.rs @@ -9,14 +9,14 @@ use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::catalog::Session; use datafusion::datasource::provider_as_source; use datafusion::logical_expr::{LogicalPlan, TableSource}; -use datafusion_common::{not_impl_err, plan_err, Result}; +use datafusion_common::{Result, not_impl_err, plan_err}; use sail_common_datafusion::datasource::{SinkInfo, SourceInfo, TableFormat}; use sail_common_datafusion::streaming::source::StreamSourceTableProvider; pub use crate::formats::socket::reader::SocketSourceExec; use crate::formats::socket::reader::SocketStreamSource; -pub use crate::options::gen::SocketReadOptions; use crate::options::ResolveOptions; +pub use crate::options::r#gen::SocketReadOptions; /// Read test data from a TCP socket for testing purposes. /// The record batches contain a single string column corresponding to lines read from the socket. diff --git a/crates/sail-data-source/src/formats/socket/options.rs b/crates/sail-data-source/src/formats/socket/options.rs index 0bee729d2..c94fdb3a2 100644 --- a/crates/sail-data-source/src/formats/socket/options.rs +++ b/crates/sail-data-source/src/formats/socket/options.rs @@ -2,7 +2,7 @@ use datafusion::catalog::Session; use sail_common_datafusion::datasource::OptionLayer; use crate::error::DataSourceResult; -use crate::options::gen::{SocketReadOptions, SocketReadPartialOptions}; +use crate::options::r#gen::{SocketReadOptions, SocketReadPartialOptions}; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; impl ResolveOptions for SocketReadOptions { diff --git a/crates/sail-data-source/src/formats/socket/reader.rs b/crates/sail-data-source/src/formats/socket/reader.rs index 1ec69a3f5..61cb660df 100644 --- a/crates/sail-data-source/src/formats/socket/reader.rs +++ b/crates/sail-data-source/src/formats/socket/reader.rs @@ -10,18 +10,18 @@ use datafusion::logical_expr::Expr; use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties}; -use datafusion_common::{exec_datafusion_err, plan_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, exec_datafusion_err, plan_err}; use futures::TryStreamExt; +use sail_common_datafusion::streaming::event::FlowEvent; use sail_common_datafusion::streaming::event::encoding::EncodedFlowEventStream; use sail_common_datafusion::streaming::event::schema::to_flow_event_schema; use sail_common_datafusion::streaming::event::stream::FlowEventStreamAdapter; -use sail_common_datafusion::streaming::event::FlowEvent; use sail_common_datafusion::streaming::source::StreamSource; use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio_stream::wrappers::LinesStream; use tokio_stream::StreamExt; +use tokio_stream::wrappers::LinesStream; -use crate::options::gen::SocketReadOptions; +use crate::options::r#gen::SocketReadOptions; #[derive(Debug, Clone)] pub struct SocketStreamSource { diff --git a/crates/sail-data-source/src/formats/text/mod.rs b/crates/sail-data-source/src/formats/text/mod.rs index 9035f743c..502fb8494 100644 --- a/crates/sail-data-source/src/formats/text/mod.rs +++ b/crates/sail-data-source/src/formats/text/mod.rs @@ -4,8 +4,8 @@ use datafusion_common::{DataFusionError, Result}; use sail_common_datafusion::datasource::OptionLayer; use crate::listing::source::{FormatFactory, ListingTableFormat}; -use crate::options::gen::{TextReadOptions, TextWriteOptions}; use crate::options::ResolveOptions; +use crate::options::r#gen::{TextReadOptions, TextWriteOptions}; pub mod options; mod read; diff --git a/crates/sail-data-source/src/formats/text/options.rs b/crates/sail-data-source/src/formats/text/options.rs index 2522571f3..25fe1995c 100644 --- a/crates/sail-data-source/src/formats/text/options.rs +++ b/crates/sail-data-source/src/formats/text/options.rs @@ -4,7 +4,7 @@ use sail_common_datafusion::datasource::OptionLayer; use crate::error::{DataSourceError, DataSourceResult}; use crate::formats::text::TableTextOptions; -use crate::options::gen::{ +use crate::options::r#gen::{ TextReadOptions, TextReadPartialOptions, TextWriteOptions, TextWritePartialOptions, }; use crate::options::{BuildPartialOptions, PartialOptions, ResolveOptions}; @@ -15,6 +15,7 @@ impl TextReadOptions { let TextReadOptions { whole_text, line_sep, + path_glob_filter: _, } = self; // Validate that line_sep (if set) is a valid ASCII byte character if let Some(c) = line_sep { @@ -88,8 +89,8 @@ mod tests { use datafusion::prelude::SessionContext; use datafusion_common::parsers::CompressionTypeVariant; - use crate::options::gen::{TextReadOptions, TextWriteOptions}; - use crate::options::{option_list, ResolveOptions}; + use crate::options::r#gen::{TextReadOptions, TextWriteOptions}; + use crate::options::{ResolveOptions, option_list}; #[test] fn test_resolve_text_read_options() -> datafusion_common::Result<()> { diff --git a/crates/sail-data-source/src/formats/text/read.rs b/crates/sail-data-source/src/formats/text/read.rs index f77f2dc0b..e1a6a34af 100644 --- a/crates/sail-data-source/src/formats/text/read.rs +++ b/crates/sail-data-source/src/formats/text/read.rs @@ -10,7 +10,7 @@ use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuil use crate::formats::text::source::TextSource; use crate::listing::source::{ListingFileSample, ListingScanInput, ReadFormat}; use crate::listing::utils::infer_listing_compression; -use crate::options::gen::TextReadOptions; +use crate::options::r#gen::TextReadOptions; #[derive(Debug, Clone)] pub struct TextReadFormat { @@ -73,4 +73,8 @@ impl ReadFormat for TextReadFormat { Ok(config) } + + fn path_glob_filter(&self) -> Option<&str> { + self.options.path_glob_filter.as_deref() + } } diff --git a/crates/sail-data-source/src/formats/text/source.rs b/crates/sail-data-source/src/formats/text/source.rs index 4b1aaa22f..40a2eacbc 100644 --- a/crates/sail-data-source/src/formats/text/source.rs +++ b/crates/sail-data-source/src/formats/text/source.rs @@ -6,16 +6,16 @@ use std::task::Poll; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::error::ArrowError; use datafusion::physical_expr::projection::ProjectionExprs; -use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::DisplayFormatType; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_common::{DataFusionError, Result}; -use datafusion_datasource::decoder::{deserialize_stream, Decoder, DecoderDeserializer}; +use datafusion_datasource::decoder::{Decoder, DecoderDeserializer, deserialize_stream}; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_compression_type::FileCompressionType; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener}; use datafusion_datasource::projection::{ProjectionOpener, SplitProjection}; -use datafusion_datasource::{calculate_range, PartitionedFile, RangeCalculation, TableSchema}; +use datafusion_datasource::{PartitionedFile, RangeCalculation, TableSchema, calculate_range}; use futures::{StreamExt, TryStreamExt}; use object_store::{GetOptions, GetResultPayload, ObjectStore}; @@ -206,7 +206,7 @@ impl FileOpener for TextOpener { RangeCalculation::Range(None) => None, RangeCalculation::Range(Some(range)) => Some(range.into()), RangeCalculation::TerminateEarly => { - return Ok(futures::stream::poll_fn(move |_| Poll::Ready(None)).boxed()) + return Ok(futures::stream::poll_fn(move |_| Poll::Ready(None)).boxed()); } }; let options = GetOptions { diff --git a/crates/sail-data-source/src/formats/text/write.rs b/crates/sail-data-source/src/formats/text/write.rs index d3827f405..51b2c17f2 100644 --- a/crates/sail-data-source/src/formats/text/write.rs +++ b/crates/sail-data-source/src/formats/text/write.rs @@ -4,14 +4,14 @@ use async_trait::async_trait; use datafusion::catalog::Session; use datafusion::logical_expr::dml::InsertOp; use datafusion::physical_plan::ExecutionPlan; -use datafusion_common::{not_impl_err, DataFusionError, GetExt, Result}; +use datafusion_common::{DataFusionError, GetExt, Result, not_impl_err}; use datafusion_datasource::file_compression_type::FileCompressionType; use datafusion_datasource::sink::DataSinkExec; -use crate::formats::text::writer::{TextSink, TextWriterOptions}; use crate::formats::text::DEFAULT_TEXT_EXTENSION; +use crate::formats::text::writer::{TextSink, TextWriterOptions}; use crate::listing::source::{ListingSinkInput, WriteFormat}; -use crate::options::gen::TextWriteOptions; +use crate::options::r#gen::TextWriteOptions; #[derive(Debug, Clone)] pub struct TextWriteFormat { diff --git a/crates/sail-data-source/src/formats/text/writer.rs b/crates/sail-data-source/src/formats/text/writer.rs index 4eefe8bcb..44eede4e8 100644 --- a/crates/sail-data-source/src/formats/text/writer.rs +++ b/crates/sail-data-source/src/formats/text/writer.rs @@ -15,9 +15,9 @@ use datafusion_common::{DataFusionError, Result}; use datafusion_datasource::display::FileGroupDisplay; use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig}; use datafusion_datasource::sink::DataSink; +use datafusion_datasource::write::BatchSerializer; use datafusion_datasource::write::demux::DemuxedStreamReceiver; use datafusion_datasource::write::orchestration::spawn_writer_tasks_and_join; -use datafusion_datasource::write::BatchSerializer; use object_store::ObjectStore; use crate::formats::text::TableTextOptions; @@ -153,7 +153,7 @@ impl DisplayAs for TextSink { } DisplayFormatType::TreeRender => { writeln!(f, "format: txt")?; - write!(f, "file={}", &self.config.original_url) + write!(f, "file={}", self.config.original_url) } } } diff --git a/crates/sail-data-source/src/listing/delete.rs b/crates/sail-data-source/src/listing/delete.rs index c46095017..38e904151 100644 --- a/crates/sail-data-source/src/listing/delete.rs +++ b/crates/sail-data-source/src/listing/delete.rs @@ -8,7 +8,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use datafusion_common::{exec_err, internal_err, Result}; +use datafusion_common::{Result, exec_err, internal_err}; use futures::{StreamExt, TryStreamExt}; use object_store::path::Path; diff --git a/crates/sail-data-source/src/listing/input_files.rs b/crates/sail-data-source/src/listing/input_files.rs new file mode 100644 index 000000000..5282242ad --- /dev/null +++ b/crates/sail-data-source/src/listing/input_files.rs @@ -0,0 +1,72 @@ +use std::any::Any; +use std::collections::HashSet; + +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::logical_expr::{LogicalPlan, TableScan}; +use datafusion::prelude::SessionContext; +use datafusion_common::{DataFusionError, Result}; +use futures::TryStreamExt; +use url::Url; + +use crate::listing::table::ListingTableSource; +use crate::listing::utils::list_all_files; + +/// Deduplicated, percent-encoded URIs of the files composing `plan` (Spark's `DataFrame.inputFiles`). +pub async fn input_files(ctx: &SessionContext, plan: LogicalPlan) -> Result> { + // Optimize first, like Spark, so eliminated scans (e.g. `WHERE false`) contribute no files. + let state = ctx.state(); + let plan = state.optimize(&plan)?; + + // Collect the listing table sources referenced anywhere in the plan. + let mut listing_sources: Vec = vec![]; + plan.apply(|node| { + if let LogicalPlan::TableScan(TableScan { source, .. }) = node { + // Upcast to `Any` to downcast to the concrete source. + let source: &dyn Any = source.as_ref(); + if let Some(listing) = source.downcast_ref::() { + listing_sources.push(listing.clone()); + } + } + Ok(TreeNodeRecursion::Continue) + })?; + + let mut files: Vec = vec![]; + // List each (path, filter) once so a repeated scan (e.g. `df.union(df)`) isn't re-listed. + let mut listed: HashSet<(String, Option)> = HashSet::new(); + + for source in &listing_sources { + let path_glob_filter = source.config().path_glob_filter.as_ref(); + for table_path in &source.config().table_paths { + if !listed.insert(( + table_path.as_str().to_string(), + path_glob_filter.map(|filter| filter.as_str().to_string()), + )) { + continue; + } + let store = ctx.runtime_env().object_store(table_path)?; + let base = Url::parse(table_path.object_store().as_str()) + .map_err(|e| DataFusionError::Internal(format!("invalid object store URL: {e}")))?; + let metas = list_all_files(table_path, &state, store.as_ref(), path_glob_filter) + .await? + .try_collect::>() + .await?; + for meta in metas { + // Percent-encode the path, as Spark returns encoded URIs. + let mut uri = base.clone(); + uri.path_segments_mut() + .map_err(|()| { + DataFusionError::Internal("object store URL cannot be a base".to_string()) + })? + .clear() + .extend(meta.location.parts().map(|part| part.as_ref().to_string())); + files.push(uri.to_string()); + } + } + } + + // Deduplicate, matching Spark's semantics. + files.sort(); + files.dedup(); + + Ok(files) +} diff --git a/crates/sail-data-source/src/listing/mod.rs b/crates/sail-data-source/src/listing/mod.rs index e87042bd3..fb05def6e 100644 --- a/crates/sail-data-source/src/listing/mod.rs +++ b/crates/sail-data-source/src/listing/mod.rs @@ -1,4 +1,5 @@ pub mod delete; +pub mod input_files; pub mod planner; pub mod source; pub mod table; diff --git a/crates/sail-data-source/src/listing/planner.rs b/crates/sail-data-source/src/listing/planner.rs index 948826a27..9caa00f1b 100644 --- a/crates/sail-data-source/src/listing/planner.rs +++ b/crates/sail-data-source/src/listing/planner.rs @@ -9,24 +9,24 @@ use datafusion::arrow::datatypes::DataType; use datafusion::catalog::Session; use datafusion::datasource::listing::helpers::pruned_partition_list; use datafusion::datasource::physical_plan::{FileOutputMode, FileSinkConfig}; -use datafusion::execution::cache::cache_manager::CachedFileMetadata; -use datafusion::execution::cache::TableScopedPath; use datafusion::execution::SessionState; +use datafusion::execution::cache::TableScopedPath; +use datafusion::execution::cache::cache_manager::CachedFileMetadata; use datafusion::logical_expr::dml::InsertOp; use datafusion::logical_expr::expr_rewriter::unnormalize_cols; use datafusion::logical_expr::{Expr, LogicalPlan, TableScan, UserDefinedLogicalNode}; use datafusion::physical_expr::create_lex_ordering; use datafusion::physical_expr_common::sort_expr::LexOrdering; -use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; use datafusion_common::stats::Precision; -use datafusion_common::{internal_err, plan_err, project_schema, Statistics}; +use datafusion_common::{Statistics, internal_err, plan_err, project_schema}; +use datafusion_datasource::ListingTableUrl; use datafusion_datasource::file_groups::FileGroup; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::source::DataSourceExec; -use datafusion_datasource::ListingTableUrl; -use futures::{future, stream, Stream, StreamExt}; +use futures::{Stream, StreamExt, TryStreamExt, future, stream}; use object_store::ObjectStore; use sail_common_datafusion::datasource::create_sort_order; use sail_common_datafusion::streaming::event::schema::is_flow_event_schema; @@ -35,7 +35,9 @@ use sail_physical_plan::barrier::BarrierExec; use crate::listing::delete::FileDeleteExec; use crate::listing::source::{ListingScanInput, ListingSinkInput}; use crate::listing::table::ListingTableSource; -use crate::listing::utils::can_be_evaluated_for_partition_pruning; +use crate::listing::utils::{ + can_be_evaluated_for_partition_pruning, has_hidden_path_component, matches_path_glob_filter, +}; use crate::listing::write::{FileWriteNode, FileWriteOptions}; /// Result of a file listing operation for listing table scans. @@ -350,16 +352,25 @@ async fn list_files_for_scan<'a>( .map(|field| (field.name().clone(), field.data_type().clone())) .collect(); - let file_list = future::try_join_all(source.config().table_paths.iter().map(|table_path| { - pruned_partition_list( - ctx, - store.as_ref(), - table_path, - filters, - "", - &partition_cols, - ) - })) + let store_ref = store.as_ref(); + let partition_cols_ref = partition_cols.as_slice(); + let path_glob_filter = source.config().path_glob_filter.as_ref(); + let file_list = future::try_join_all(source.config().table_paths.iter().map( + |table_path| async move { + let files = + pruned_partition_list(ctx, store_ref, table_path, filters, "", partition_cols_ref) + .await?; + // Apply listing-level file-name filters so scans agree with `inputFiles` and sampling. + let files = files.try_filter(move |file| { + let location = &file.object_meta.location; + futures::future::ready( + !has_hidden_path_component(table_path, location) + && matches_path_glob_filter(path_glob_filter, location), + ) + }); + Ok::<_, datafusion_common::DataFusionError>(files.boxed()) + }, + )) .await?; let meta_fetch_concurrency = ctx.config_options().execution.meta_fetch_concurrency; @@ -522,24 +533,21 @@ async fn get_files_with_limit( let file = file_result?; - if collect_stats { - if let Some(file_stats) = &file.statistics { - num_rows = if file_group.is_empty() { - file_stats.num_rows - } else { - num_rows.add(&file_stats.num_rows) - }; - } + if collect_stats && let Some(file_stats) = &file.statistics { + num_rows = if file_group.is_empty() { + file_stats.num_rows + } else { + num_rows.add(&file_stats.num_rows) + }; } file_group.push(file); - if let Some(limit) = limit { - if let Precision::Exact(row_count) = num_rows { - if row_count > limit { - state = ProcessingState::ReachedLimit; - } - } + if let Some(limit) = limit + && let Precision::Exact(row_count) = num_rows + && row_count > limit + { + state = ProcessingState::ReachedLimit; } } diff --git a/crates/sail-data-source/src/listing/source.rs b/crates/sail-data-source/src/listing/source.rs index df5d13975..8a7f69b35 100644 --- a/crates/sail-data-source/src/listing/source.rs +++ b/crates/sail-data-source/src/listing/source.rs @@ -12,15 +12,15 @@ use datafusion::physical_expr::LexRequirement; use datafusion::physical_expr_common::sort_expr::LexOrdering; use datafusion::physical_plan::ExecutionPlan; use datafusion_common::parsers::CompressionTypeVariant; -use datafusion_common::{not_impl_err, plan_err, Result, Statistics}; +use datafusion_common::{Result, Statistics, not_impl_err, plan_err}; use datafusion_datasource::file_groups::FileGroup; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::{ListingTableUrl, TableSchema}; use futures::TryStreamExt; use object_store::{ObjectMeta, ObjectStore}; use sail_common_datafusion::datasource::{ - find_path_in_options, get_partition_columns_and_file_schema, OptionLayer, SinkInfo, SinkMode, - SourceInfo, TableFormat, + OptionLayer, SinkInfo, SinkMode, SourceInfo, TableFormat, find_path_in_options, + get_partition_columns_and_file_schema, }; use url::Url; @@ -30,7 +30,7 @@ use crate::listing::utils::{ }; use crate::listing::write::{FileWriteNode, FileWriteOptions}; use crate::resolve_listing_urls; -use crate::url::resolve_listing_writer_url; +use crate::url::{PathGlobFilter, resolve_listing_writer_url}; /// A trait for creating format instances when reading and writing listing files. pub trait FormatFactory: Debug + Send + Sync + 'static { @@ -83,6 +83,21 @@ pub trait ReadFormat: Debug + Send + Sync + 'static { /// Build a scan configuration for listing reads. async fn scan(&self, ctx: &dyn Session, input: ListingScanInput) -> Result; + + /// Whether validating an explicit schema requires the physical file schema. + fn requires_explicit_schema_validation(&self) -> bool { + false + } + + /// Validate a user-provided file schema against the physical file schema. + fn validate_explicit_schema(&self, _schema: &Schema, _physical: &Schema) -> Result<()> { + Ok(()) + } + + /// File-name glob restricting which listed files compose the dataset. + fn path_glob_filter(&self) -> Option<&str> { + None + } } #[derive(Debug)] @@ -159,23 +174,38 @@ impl TableFormat for ListingTableFormat { } = info; let read_format = T::read(ctx, options)?; + let path_glob_filter = read_format + .path_glob_filter() + .map(PathGlobFilter::parse) + .transpose()?; let urls = resolve_listing_urls(ctx, paths).await?; - let sampled_files = sample_listing_files(ctx, &urls).await?; + let sampled_files = sample_listing_files(ctx, &urls, path_glob_filter.as_ref()).await?; let compression = read_format.infer_compression(ctx, &sampled_files).await?; let (schema, partition_fields) = match schema { Some(schema) if !schema.fields().is_empty() => { + let physical = if read_format.requires_explicit_schema_validation() { + Some( + read_format + .infer_schema(ctx, &sampled_files, compression) + .await?, + ) + } else if read_case_sensitive { + None + } else { + read_format + .infer_schema(ctx, &sampled_files, compression) + .await + .ok() + }; // Spark matches a user-specified schema against the physical file // columns case-insensitively by default (`spark.sql.caseSensitive=false`). // Reconcile the user column names to the physical names up front so that both // the file stats and reader (which resolve columns by exact name) find the data. let schema = if read_case_sensitive { schema - } else if let Ok(physical) = read_format - .infer_schema(ctx, &sampled_files, compression) - .await - { - reconcile_schema_names_case_insensitive(schema, &physical)? + } else if let Some(physical) = &physical { + reconcile_schema_names_case_insensitive(schema, physical)? } else { // Keeps the user schema if physical schema inference is unavailable. schema @@ -200,6 +230,9 @@ impl TableFormat for ListingTableFormat { }; let (partition_fields, schema) = get_partition_columns_and_file_schema(&schema, partition_by)?; + if let Some(physical) = physical { + read_format.validate_explicit_schema(&schema, &physical)?; + } (Arc::new(schema), partition_fields) } _ => { @@ -234,6 +267,7 @@ impl TableFormat for ListingTableFormat { collect_stat: ctx.config().collect_statistics(), target_partitions: ctx.config().target_partitions(), read_format: Arc::new(read_format), + path_glob_filter, compression, })?; Ok(Arc::new(source)) @@ -296,12 +330,11 @@ impl TableFormat for ListingTableFormat { } async fn listing_target_exists(ctx: &dyn Session, url: &Url) -> Result { // For file systems, treat the target as existing even if it is an empty directory. - if url.scheme() == "file" { - if let Ok(path) = url.to_file_path() { - if path.exists() { - return Ok(true); - } - } + if url.scheme() == "file" + && let Ok(path) = url.to_file_path() + && path.exists() + { + return Ok(true); } listing_target_nonempty(ctx, url).await } diff --git a/crates/sail-data-source/src/listing/table.rs b/crates/sail-data-source/src/listing/table.rs index 2ef187aec..10189151e 100644 --- a/crates/sail-data-source/src/listing/table.rs +++ b/crates/sail-data-source/src/listing/table.rs @@ -12,6 +12,7 @@ use datafusion_datasource::{ListingTableUrl, TableSchema}; use crate::listing::source::ReadFormat; use crate::listing::utils::can_be_evaluated_for_partition_pruning; +use crate::url::PathGlobFilter; #[derive(Clone, Debug)] pub struct ListingTableSourceConfig { @@ -22,6 +23,7 @@ pub struct ListingTableSourceConfig { pub collect_stat: bool, pub target_partitions: usize, pub read_format: Arc, + pub path_glob_filter: Option, pub compression: CompressionTypeVariant, } diff --git a/crates/sail-data-source/src/listing/utils.rs b/crates/sail-data-source/src/listing/utils.rs index 0e67ab1e3..f35176e8f 100644 --- a/crates/sail-data-source/src/listing/utils.rs +++ b/crates/sail-data-source/src/listing/utils.rs @@ -3,20 +3,22 @@ use std::sync::Arc; use arrow_schema::FieldRef; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::datasource::listing::helpers::expr_applicable_for_cols; -use datafusion::execution::cache::cache_manager::CachedFileList; use datafusion::execution::cache::TableScopedPath; +use datafusion::execution::cache::cache_manager::CachedFileList; use datafusion::logical_expr::Expr; use datafusion_common::parsers::CompressionTypeVariant; -use datafusion_common::{internal_datafusion_err, plan_err, DataFusionError, GetExt, Result}; -use datafusion_datasource::file_compression_type::FileCompressionType; +use datafusion_common::{DataFusionError, GetExt, Result, internal_datafusion_err, plan_err}; use datafusion_datasource::ListingTableUrl; +use datafusion_datasource::file_compression_type::FileCompressionType; use datafusion_session::Session; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use log::debug; +use object_store::path::Path; use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; use crate::listing::source::ListingFileSample; +use crate::url::PathGlobFilter; pub fn rewrite_utf8view_fields(schema: Arc) -> Arc { // TODO: Spark doesn't support Utf8View @@ -88,11 +90,12 @@ pub fn infer_listing_compression( pub async fn sample_listing_files<'a>( ctx: &dyn Session, urls: &'a [ListingTableUrl], + path_glob_filter: Option<&'a PathGlobFilter>, ) -> Result>> { let mut samples = vec![]; for url in urls { let store = ctx.runtime_env().object_store(url)?; - let objects: Vec<_> = list_all_files(url, ctx, store.as_ref()) + let objects: Vec<_> = list_all_files(url, ctx, store.as_ref(), path_glob_filter) .await? // Empty files can't contribute to schema / partition inference and may error when read. .try_filter(|meta| futures::future::ready(meta.size > 0)) @@ -197,6 +200,7 @@ pub async fn list_all_files<'a>( url: &'a ListingTableUrl, ctx: &'a dyn Session, store: &'a dyn ObjectStore, + path_glob_filter: Option<&'a PathGlobFilter>, ) -> Result>> { let exec_options = &ctx.config_options().execution; let ignore_subdirectory = exec_options.listing_table_ignore_subdirectory; @@ -225,16 +229,93 @@ pub async fn list_all_files<'a>( Ok(list .try_filter(move |meta| { let path = &meta.location; - let glob_match = url.contains(path, ignore_subdirectory); - futures::future::ready(glob_match) + let included = url.contains(path, ignore_subdirectory) + && !has_hidden_path_component(url, path) + && matches_path_glob_filter(path_glob_filter, path); + futures::future::ready(included) }) .map_err(|e| DataFusionError::ObjectStore(Box::new(e))) .boxed()) } +pub fn matches_path_glob_filter( + path_glob_filter: Option<&PathGlobFilter>, + location: &Path, +) -> bool { + path_glob_filter.is_none_or(|filter| { + location + .filename() + .is_some_and(|filename| filter.matches(filename)) + }) +} + +/// Returns `true` if the path is hidden per Spark's `HadoopFSUtils.shouldFilterOutPathName`. +pub fn has_hidden_path_component(url: &ListingTableUrl, location: &Path) -> bool { + let is_hidden = |name: &str| { + let exclude = (name.starts_with('_') && !name.contains('=')) + || name.starts_with('.') + || name.ends_with("._COPYING_"); + let keep = name.starts_with("_common_metadata") || name.starts_with("_metadata"); + exclude && !keep + }; + url.strip_prefix(location) + .is_some_and(|mut segments| segments.any(is_hidden)) + || location.filename().is_some_and(is_hidden) +} + pub fn can_be_evaluated_for_partition_pruning( partition_column_names: &[&str], expr: &Expr, ) -> bool { !partition_column_names.is_empty() && expr_applicable_for_cols(partition_column_names, expr) } + +#[expect(clippy::unwrap_used)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_has_hidden_path_component() { + let dir = ListingTableUrl::parse("file:///data/").unwrap(); + let hidden = |path: &str| has_hidden_path_component(&dir, &Path::from(path)); + + // Data files and partition directories are kept. + assert!(!hidden("data/part-0.parquet")); + assert!(!hidden("data/year=2020/part-0.parquet")); + + // Hidden markers and hidden directories are excluded. + assert!(hidden("data/_SUCCESS")); + assert!(hidden("data/.hidden.json")); + assert!(hidden("data/_temporary/0/part-0.parquet")); + assert!(hidden("data/visible/_hidden/bad.json")); + + // Files mid-copy (Hadoop `._COPYING_`) are excluded. + assert!(hidden("data/part-0.parquet._COPYING_")); + + // Spark keeps the Parquet summary files. + assert!(!hidden("data/_metadata")); + assert!(!hidden("data/_common_metadata")); + + // Spark keeps `_`-prefixed partition directories (they contain `=`). + assert!(!hidden("data/_part=1/part-0.parquet")); + + // A hidden listing root is not itself filtered. + let hidden_root = ListingTableUrl::parse("file:///_root/").unwrap(); + assert!(!has_hidden_path_component( + &hidden_root, + &Path::from("_root/part-0.parquet") + )); + + // A location outside the prefix falls back to judging its file name. + assert!(!hidden("outside/part-0.parquet")); + assert!(hidden("outside/_SUCCESS")); + + // An explicitly targeted hidden file is excluded. + let file = ListingTableUrl::parse("file:///data/_data.json").unwrap(); + assert!(has_hidden_path_component( + &file, + &Path::from("data/_data.json") + )); + } +} diff --git a/crates/sail-data-source/src/options/mod.rs b/crates/sail-data-source/src/options/mod.rs index 60ff08c3a..026543835 100644 --- a/crates/sail-data-source/src/options/mod.rs +++ b/crates/sail-data-source/src/options/mod.rs @@ -5,7 +5,7 @@ use sail_common_datafusion::datasource::OptionLayer; use crate::error::DataSourceResult; -pub mod gen { +pub mod r#gen { include!(concat!(env!("OUT_DIR"), "/options/socket.rs")); include!(concat!(env!("OUT_DIR"), "/options/rate.rs")); include!(concat!(env!("OUT_DIR"), "/options/console.rs")); diff --git a/crates/sail-data-source/src/url.rs b/crates/sail-data-source/src/url.rs index f63ac6d36..3f54e4580 100644 --- a/crates/sail-data-source/src/url.rs +++ b/crates/sail-data-source/src/url.rs @@ -6,7 +6,7 @@ use chumsky::text::whitespace; use chumsky::{IterParser, Parser}; use datafusion::catalog::Session; use datafusion::datasource::listing::ListingTableUrl; -use datafusion_common::{not_impl_err, plan_datafusion_err, plan_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, not_impl_err, plan_datafusion_err, plan_err}; use glob::Pattern; use log::debug; use object_store::ObjectStoreExt; @@ -224,6 +224,14 @@ impl PatternSegment { } } +fn expand_glob_pattern(pattern: &str) -> Result> { + let patterns = PatternSegment::sequence_parser() + .parse(pattern) + .into_result() + .map_err(|_| plan_datafusion_err!("glob pattern: {pattern}"))?; + Ok(PatternSegment::expand_sequence(patterns)) +} + /// A parsed URL that may contain glob patterns. /// The parsing is coarse-grained and permissive since we only need to /// identify the path that may contain glob patterns. @@ -389,11 +397,7 @@ impl GlobUrl { } fn parse_glob_path(path: &str) -> Result)>> { - let patterns = PatternSegment::sequence_parser() - .parse(path) - .into_result() - .map_err(|_| plan_datafusion_err!("glob path: {path}"))?; - let paths = PatternSegment::expand_sequence(patterns) + let paths = expand_glob_pattern(path)? .into_iter() .map(|x| { let (prefix, suffix) = Self::split_glob_path(&x)?; @@ -475,6 +479,39 @@ impl GlobUrl { } } +#[derive(Clone, Debug, PartialEq)] +pub struct PathGlobFilter { + raw: String, + patterns: Vec, +} + +impl PathGlobFilter { + pub fn parse(pattern: &str) -> Result { + let parse = || -> Result { + let patterns = expand_glob_pattern(pattern)? + .into_iter() + .map(|pattern| { + GlobUrl::create_percent_decoded_pattern(&pattern)? + .ok_or_else(|| plan_datafusion_err!("path glob filter cannot be empty")) + }) + .collect::>>()?; + Ok(Self { + raw: pattern.to_string(), + patterns, + }) + }; + parse().map_err(|e| DataFusionError::Plan(format!("invalid path glob filter: {e}"))) + } + + pub fn as_str(&self) -> &str { + &self.raw + } + + pub fn matches(&self, value: &str) -> bool { + self.patterns.iter().any(|pattern| pattern.matches(value)) + } +} + impl AsRef for GlobUrl { fn as_ref(&self) -> &Url { &self.base @@ -746,6 +783,18 @@ mod tests { test("{a,b{c,d}{e,f}}", &["a", "bce", "bcf", "bde", "bdf"]); } + #[test] + fn test_path_glob_filter() { + let filter = PathGlobFilter::parse("*.{json,csv}").unwrap(); + assert!(filter.matches("data.json")); + assert!(filter.matches("data.csv")); + assert!(!filter.matches("data.parquet")); + assert_eq!(filter.as_str(), "*.{json,csv}"); + + let error = PathGlobFilter::parse("[").unwrap_err(); + assert!(error.to_string().contains("invalid path glob filter")); + } + #[test] fn test_parse_glob_url() { fn test(input: &str, expected: &[(&str, Option<&str>)]) { @@ -769,6 +818,11 @@ mod tests { &[("https://example.com/path/?foo#bar", Some("**/*.txt"))], ); test("s3://bucket/path", &[("s3://bucket/path", None)]); + test("oss://bucket/path", &[("oss://bucket/path", None)]); + test( + "oss://bucket/path/*.txt", + &[("oss://bucket/path/", Some("*.txt"))], + ); test( "s3://bucket/path/*.txt", &[("s3://bucket/path/", Some("*.txt"))], diff --git a/crates/sail-data-source/src/utils.rs b/crates/sail-data-source/src/utils.rs index 27172fef5..0e60b7a91 100644 --- a/crates/sail-data-source/src/utils.rs +++ b/crates/sail-data-source/src/utils.rs @@ -1,4 +1,4 @@ -use datafusion_common::{plan_err, DataFusionError}; +use datafusion_common::{DataFusionError, plan_err}; pub fn char_to_u8(c: char, option: &str) -> datafusion_common::Result { if c.is_ascii() { diff --git a/crates/sail-delta-lake/src/catalog_managed.rs b/crates/sail-delta-lake/src/catalog_managed.rs index 3b4a4be4c..7aa728287 100644 --- a/crates/sail-delta-lake/src/catalog_managed.rs +++ b/crates/sail-delta-lake/src/catalog_managed.rs @@ -1,5 +1,5 @@ use sail_common_datafusion::catalog::delta::{ - unity_table_id_value, DELTA_TABLE_TYPE_VALUE, DELTA_UNITY_TABLE_ID_KEY, + DELTA_TABLE_TYPE_VALUE, DELTA_UNITY_TABLE_ID_KEY, unity_table_id_value, }; use sail_common_datafusion::catalog::{CatalogPartitionField, TableColumnStatus, TableKind}; diff --git a/crates/sail-delta-lake/src/checkpoint/action_fields.rs b/crates/sail-delta-lake/src/checkpoint/action_fields.rs index cfbdda61b..ff739b514 100644 --- a/crates/sail-delta-lake/src/checkpoint/action_fields.rs +++ b/crates/sail-delta-lake/src/checkpoint/action_fields.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use datafusion::arrow::array::{new_null_array, Array, ArrayRef, StringArray, StructArray}; +use datafusion::arrow::array::{Array, ArrayRef, StringArray, StructArray, new_null_array}; use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::{ DataType as ArrowDataType, Field, FieldRef, Fields, Schema as ArrowSchema, @@ -14,9 +14,9 @@ use crate::schema::make_physical_arrow_schema; use crate::snapshot::materialize::parse_partition_values_array; use crate::spec::fields::{FIELD_NAME_PARTITION_VALUES_PARSED, FIELD_NAME_STATS_PARSED}; use crate::spec::{ - add_struct_type, parse_stats_json_array, remove_struct_type, stats_schema, ColumnMappingMode, - ColumnMetadataKey, DeltaError as DeltaTableError, DeltaResult, Metadata, StructField, - StructType, TableProperties, + ColumnMappingMode, ColumnMetadataKey, DeltaError as DeltaTableError, DeltaResult, Metadata, + StructField, StructType, TableProperties, add_struct_type, parse_stats_json_array, + remove_struct_type, stats_schema, }; pub(crate) struct AddAugmentationConfig { @@ -156,16 +156,16 @@ impl AddAugmentationConfig { add_fields.push(Arc::new(Field::new( FIELD_NAME_PARTITION_VALUES_PARSED, partition_values_parsed.data_type().clone(), - false, + true, ))); add_cols.push(Arc::new(partition_values_parsed)); } - if !self.write_stats_as_json { - if let Some(stats_idx) = stats_idx { - add_fields.remove(stats_idx); - add_cols.remove(stats_idx); - } + if !self.write_stats_as_json + && let Some(stats_idx) = stats_idx + { + add_fields.remove(stats_idx); + add_cols.remove(stats_idx); } let new_add = StructArray::try_new(Fields::from(add_fields), add_cols, nulls)?; diff --git a/crates/sail-delta-lake/src/checkpoint/mod.rs b/crates/sail-delta-lake/src/checkpoint/mod.rs index ddf7bc9f1..daf22b89b 100644 --- a/crates/sail-delta-lake/src/checkpoint/mod.rs +++ b/crates/sail-delta-lake/src/checkpoint/mod.rs @@ -28,28 +28,29 @@ use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::runtime::SpawnedTask; use log::debug; use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; +use parquet::arrow::AsyncArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::async_writer::ParquetObjectWriter; -use parquet::arrow::AsyncArrowWriter; use uuid::Uuid; use crate::checkpoint::action_fields::{ - normalize_checkpoint_batch_for_decode, AddAugmentationConfig, + AddAugmentationConfig, normalize_checkpoint_batch_for_decode, }; use crate::delta_log::segment_files::ReplayedTableHeader; use crate::delta_log::{ - get_actions, list_delta_log_entries_from, parse_checkpoint_version_from_location, + LogStore, get_actions, list_delta_log_entries_from, parse_checkpoint_version_from_location, parse_commit_version_from_location, read_last_checkpoint_version_from_store, - resolve_commit_timestamp_from_actions, LogStore, + resolve_commit_timestamp_from_actions, }; pub(crate) use crate::delta_log::{ latest_replayable_version, load_replayed_table_header, load_replayed_table_state, }; use crate::spec::{ - checkpoint_path, is_json_checkpoint_filename, last_checkpoint_path, logical_file_key, - sidecar_file_path, uuid_checkpoint_path, Action, Add, CheckpointActionRow, CheckpointMetadata, - DeltaError as DeltaTableError, DeltaResult, DomainMetadata, LastCheckpointHint, LogicalFileKey, - Metadata, Protocol, Remove, Sidecar, TableFeature, TableProperties, Transaction, + Action, Add, CheckpointActionRow, CheckpointMetadata, DeltaError as DeltaTableError, + DeltaResult, DomainMetadata, LastCheckpointHint, LogicalFileKey, Metadata, Protocol, Remove, + Sidecar, TableFeature, TableProperties, Transaction, checkpoint_path, + is_json_checkpoint_filename, last_checkpoint_path, logical_file_key, sidecar_file_path, + uuid_checkpoint_path, }; mod action_fields; @@ -588,10 +589,10 @@ impl<'a> CheckpointManager<'a> { } continue; } - if let Some(v) = parse_checkpoint_version_from_location(&meta.location) { - if v <= version { - checkpoint_entries.push((v, meta)); - } + if let Some(v) = parse_checkpoint_version_from_location(&meta.location) + && v <= version + { + checkpoint_entries.push((v, meta)); } } commit_entries.sort_by_key(|(av, _)| *av); @@ -717,32 +718,34 @@ impl<'a> CheckpointManager<'a> { augment: sidecar_augment, }; - let sidecar_descriptor = if let Some(first_batch) = sidecar_batches.next_batch()? { - ensure_schema_supported_for_parquet(&first_batch)?; - let sidecar_writer = ParquetObjectWriter::new(store.clone(), sidecar_path.clone()); - let mut writer = AsyncArrowWriter::try_new(sidecar_writer, first_batch.schema(), None) - .map_err(DeltaTableError::generic_err)?; - writer - .write(&first_batch) - .await - .map_err(DeltaTableError::generic_err)?; - while let Some(batch) = sidecar_batches.next_batch()? { + let sidecar_descriptor = match sidecar_batches.next_batch()? { + Some(first_batch) => { + ensure_schema_supported_for_parquet(&first_batch)?; + let sidecar_writer = ParquetObjectWriter::new(store.clone(), sidecar_path.clone()); + let mut writer = + AsyncArrowWriter::try_new(sidecar_writer, first_batch.schema(), None) + .map_err(DeltaTableError::generic_err)?; writer - .write(&batch) + .write(&first_batch) .await .map_err(DeltaTableError::generic_err)?; + while let Some(batch) = sidecar_batches.next_batch()? { + writer + .write(&batch) + .await + .map_err(DeltaTableError::generic_err)?; + } + let _ = writer.close().await.map_err(DeltaTableError::generic_err)?; + let sidecar_meta = store.head(&sidecar_path).await?; + Some(Sidecar { + path: sidecar_filename, + size_in_bytes: i64::try_from(sidecar_meta.size) + .map_err(|_| DeltaTableError::generic("sidecar size overflow"))?, + modification_time: now_millis, + tags: None, + }) } - let _ = writer.close().await.map_err(DeltaTableError::generic_err)?; - let sidecar_meta = store.head(&sidecar_path).await?; - Some(Sidecar { - path: sidecar_filename, - size_in_bytes: i64::try_from(sidecar_meta.size) - .map_err(|_| DeltaTableError::generic("sidecar size overflow"))?, - modification_time: now_millis, - tags: None, - }) - } else { - None + _ => None, }; // Step 2: Build the main V2 checkpoint with header actions + sidecar refs. @@ -1383,22 +1386,22 @@ mod tests { use object_store::memory::InMemory; use object_store::path::Path; use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; - use parquet::arrow::async_writer::ParquetObjectWriter; use parquet::arrow::AsyncArrowWriter; + use parquet::arrow::async_writer::ParquetObjectWriter; use super::{ - checkpoint_fields, decode_checkpoint_rows, encode_checkpoint_rows, - read_checkpoint_rows_from_checkpoint_file, replay_commit_header_actions, - ReconciledCheckpointState, ReconciledHeaderState, + ReconciledCheckpointState, ReconciledHeaderState, checkpoint_fields, + decode_checkpoint_rows, encode_checkpoint_rows, read_checkpoint_rows_from_checkpoint_file, + replay_commit_header_actions, }; use crate::checkpoint::action_fields::{ - normalize_checkpoint_batch_for_decode, AddAugmentationConfig, + AddAugmentationConfig, normalize_checkpoint_batch_for_decode, }; use crate::spec::{ - sidecar_file_path, Action, Add, CheckpointActionRow, CheckpointMetadata, CommitInfo, - DataType, DeletionVectorDescriptor, DeltaError as DeltaTableError, DeltaResult, - DomainMetadata, Metadata, Protocol, Remove, Sidecar, StorageType, StructField, StructType, - TableFeature, Transaction, + Action, Add, CheckpointActionRow, CheckpointMetadata, CommitInfo, DataType, + DeletionVectorDescriptor, DeltaError as DeltaTableError, DeltaResult, DomainMetadata, + Metadata, Protocol, Remove, Sidecar, StorageType, StructField, StructType, TableFeature, + Transaction, sidecar_file_path, }; fn encode_rows_for_test(rows: &Vec) -> DeltaResult { @@ -1976,8 +1979,7 @@ mod tests { commit_timestamp: None, }; - let sidecar_filename = - "00000000000000000002.checkpoint.0000000001.0000000001.bbf4d2d5-b626-41f8-854f-63b5e397ad82.parquet"; + let sidecar_filename = "00000000000000000002.checkpoint.0000000001.0000000001.bbf4d2d5-b626-41f8-854f-63b5e397ad82.parquet"; let sidecar_batch = encode_rows_for_test(&vec![CheckpointActionRow { add: Some(add.clone()), ..Default::default() @@ -2160,15 +2162,21 @@ mod tests { state.prune_expired_checkpoint_actions(now)?; - assert!(!state - .removes - .contains_key(&("expired.parquet".to_string(), None))); - assert!(state - .removes - .contains_key(&("fresh.parquet".to_string(), None))); - assert!(state - .removes - .contains_key(&("unknown-ts.parquet".to_string(), None))); + assert!( + !state + .removes + .contains_key(&("expired.parquet".to_string(), None)) + ); + assert!( + state + .removes + .contains_key(&("fresh.parquet".to_string(), None)) + ); + assert!( + state + .removes + .contains_key(&("unknown-ts.parquet".to_string(), None)) + ); assert!(!state.txns.contains_key("expired-app")); assert!(state.txns.contains_key("fresh-app")); assert!(state.txns.contains_key("legacy-app")); @@ -2205,9 +2213,11 @@ mod tests { state.prune_expired_checkpoint_actions(now)?; - assert!(state - .removes - .contains_key(&("older-remove.parquet".to_string(), None))); + assert!( + state + .removes + .contains_key(&("older-remove.parquet".to_string(), None)) + ); Ok(()) } diff --git a/crates/sail-delta-lake/src/conversion/mod.rs b/crates/sail-delta-lake/src/conversion/mod.rs index 200ca6863..b6939177d 100644 --- a/crates/sail-delta-lake/src/conversion/mod.rs +++ b/crates/sail-delta-lake/src/conversion/mod.rs @@ -2,7 +2,7 @@ pub mod scalar; pub mod type_promotion; pub use scalar::{ - parse_optional_partition_value, parse_partition_value, scalar_from_array_opt, - scalar_value_to_array, ScalarConverter, ScalarExt, + ScalarConverter, ScalarExt, parse_optional_partition_value, parse_partition_value, + scalar_from_array_opt, scalar_value_to_array, }; pub use type_promotion::DeltaTypeConverter; diff --git a/crates/sail-delta-lake/src/conversion/scalar.rs b/crates/sail-delta-lake/src/conversion/scalar.rs index f45818c92..5426d4e4a 100644 --- a/crates/sail-delta-lake/src/conversion/scalar.rs +++ b/crates/sail-delta-lake/src/conversion/scalar.rs @@ -24,15 +24,15 @@ use std::sync::Arc; use chrono::{DateTime, TimeZone, Utc}; use datafusion::arrow::array::{ - Array, ArrayRef, BooleanArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, - Int8Array, LargeStringArray, RecordBatch, StringArray, UInt16Array, UInt32Array, UInt64Array, - UInt8Array, + Array, ArrayRef, BooleanArray, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, + Int64Array, LargeStringArray, RecordBatch, StringArray, UInt8Array, UInt16Array, UInt32Array, + UInt64Array, }; -use datafusion::arrow::compute::{cast, cast_with_options, CastOptions}; +use datafusion::arrow::compute::{CastOptions, cast, cast_with_options}; use datafusion::arrow::datatypes::{DataType as ArrowDataType, TimeUnit}; -use datafusion::common::scalar::ScalarValue; use datafusion::common::Result as DataFusionResult; -use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC}; +use datafusion::common::scalar::ScalarValue; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; use serde_json::Value; use crate::spec::{DeltaError as DeltaTableError, DeltaResult as DeltaResultLocal, StatValue}; @@ -54,7 +54,7 @@ impl ScalarConverter { field_dt: &ArrowDataType, ) -> DataFusionResult> { macro_rules! typed_array { - ($array_ty:ty, $extract:expr) => {{ + ($array_ty:ty, $extract:expr_2021) => {{ let mut out = Vec::with_capacity(values.len()); for value in values { match value { @@ -555,7 +555,7 @@ mod tests { use datafusion::common::ScalarValue; use super::{ - parse_optional_partition_value, parse_partition_value, NULL_PARTITION_VALUE_DATA_PATH, + NULL_PARTITION_VALUE_DATA_PATH, parse_optional_partition_value, parse_partition_value, }; #[test] diff --git a/crates/sail-delta-lake/src/conversion/type_promotion.rs b/crates/sail-delta-lake/src/conversion/type_promotion.rs index a69bc2f18..a066883b6 100644 --- a/crates/sail-delta-lake/src/conversion/type_promotion.rs +++ b/crates/sail-delta-lake/src/conversion/type_promotion.rs @@ -15,8 +15,8 @@ use std::sync::Arc; use datafusion::arrow::compute::can_cast_types; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::common::Result; -use datafusion::physical_expr::expressions::CastExpr; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::expressions::CastExpr; use datafusion_common::DataFusionError; use indexmap::IndexMap; @@ -115,7 +115,7 @@ impl DeltaTypeConverter { "Unexpected map child layout for field '{}': {:?}", table_field.name(), t_kv_dt - ))) + ))); } }; let (i_key, i_val) = match i_kv_dt { @@ -127,7 +127,7 @@ impl DeltaTypeConverter { "Unexpected map child layout for input field '{}': {:?}", table_field.name(), i_kv_dt - ))) + ))); } }; let merged_key = Self::promote_field_types(t_key, i_key)?; @@ -221,8 +221,8 @@ impl DeltaTypeConverter { DataType::Decimal128(to_precision, to_scale), ) if (from_precision > to_precision || from_scale > to_scale) => { return Err(DataFusionError::Plan(format!( - "Potential precision loss in decimal cast for field '{field_name}': from Decimal({from_precision},{from_scale}) to Decimal({to_precision},{to_scale})" - ))); + "Potential precision loss in decimal cast for field '{field_name}': from Decimal({from_precision},{from_scale}) to Decimal({to_precision},{to_scale})" + ))); } (DataType::Int64, DataType::Int32) | (DataType::Int32, DataType::Int16) diff --git a/crates/sail-delta-lake/src/datasource/expressions.rs b/crates/sail-delta-lake/src/datasource/expressions.rs index 894145c85..bcb4eb987 100644 --- a/crates/sail-delta-lake/src/datasource/expressions.rs +++ b/crates/sail-delta-lake/src/datasource/expressions.rs @@ -19,16 +19,21 @@ use std::collections::HashSet; use std::sync::Arc; +use datafusion::arrow::datatypes::{DataType as ArrowDataType, FieldRef, Schema as ArrowSchema}; use datafusion::catalog::Session; use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; -use datafusion::common::{Column, DFSchema, Result}; +use datafusion::common::{Column, DFSchema, Result, ScalarValue}; +use datafusion::functions::core::getfield::GetFieldFunc; use datafusion::logical_expr::simplify::SimplifyContextBuilder; use datafusion::logical_expr::{BinaryExpr, Expr, Operator, TableProviderFilterPushDown}; use datafusion::optimizer::simplify_expressions::ExprSimplifier; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_plan::expressions::Column as PhysicalColumn; +use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; +use datafusion::physical_plan::expressions::{ + Column as PhysicalColumn, Literal as PhysicalLiteral, +}; -use crate::spec::DeltaResult; +use crate::schema::arrow_field_physical_name; +use crate::spec::{ColumnMappingMode, DeltaResult}; // [Credit]: @@ -70,7 +75,7 @@ pub fn get_pushdown_filters( fn expr_is_exact_predicate_for_cols(partition_cols: &[String], expr: &Expr) -> bool { let mut is_applicable = true; let _ = expr.apply(|expr| match expr { - Expr::Column(Column { ref name, .. }) => { + Expr::Column(Column { name, .. }) => { is_applicable &= partition_cols.contains(name); if is_applicable { @@ -79,7 +84,7 @@ fn expr_is_exact_predicate_for_cols(partition_cols: &[String], expr: &Expr) -> b Ok(TreeNodeRecursion::Stop) } } - Expr::BinaryExpr(BinaryExpr { ref op, .. }) => { + Expr::BinaryExpr(BinaryExpr { op, .. }) => { is_applicable &= matches!( op, Operator::And @@ -111,6 +116,122 @@ fn expr_is_exact_predicate_for_cols(partition_cols: &[String], expr: &Expr) -> b is_applicable } +/// Rewrite column references in a parquet pushdown predicate from logical names to the +/// physical names used in the data files of column-mapped tables. Nested struct field +/// accesses (`get_field`) are rewritten as well, using the column mapping metadata carried +/// by the logical schema. Partition columns are exposed to the file scan as virtual columns +/// under their logical names and are left untouched. +pub fn rewrite_predicate_for_column_mapping( + expr: Arc, + logical_schema: &ArrowSchema, + mode: ColumnMappingMode, + partition_cols: &[String], +) -> Result> { + if mode == ColumnMappingMode::None { + return Ok(expr); + } + Ok(rewrite_expr_for_column_mapping(expr, logical_schema, mode, partition_cols)?.0) +} + +/// Recursively rewrite an expression, returning the rewritten expression along with the +/// logical schema field it resolves to (when the expression is a column or a chain of +/// struct field accesses rooted at a column). +fn rewrite_expr_for_column_mapping( + expr: Arc, + logical_schema: &ArrowSchema, + mode: ColumnMappingMode, + partition_cols: &[String], +) -> Result<(Arc, Option)> { + if let Some(column) = expr.downcast_ref::() { + if partition_cols.iter().any(|col| col == column.name()) { + return Ok((expr, None)); + } + let Some((_, field)) = logical_schema.fields().find(column.name()) else { + return Ok((expr, None)); + }; + let physical_name = arrow_field_physical_name(field, mode); + let rewritten: Arc = if physical_name == column.name() { + Arc::clone(&expr) + } else { + // Column mapping changes names, not file order, so the logical index remains valid. + Arc::new(PhysicalColumn::new(physical_name, column.index())) + }; + return Ok((rewritten, Some(Arc::clone(field)))); + } + + if ScalarFunctionExpr::try_downcast_func::(expr.as_ref()).is_some() { + let children = expr.children(); + let Some((source_expr, path_exprs)) = children.split_first() else { + return Ok((expr, None)); + }; + let (new_source, source_field) = rewrite_expr_for_column_mapping( + Arc::clone(source_expr), + logical_schema, + mode, + partition_cols, + )?; + let mut current_field = source_field; + // Rewriters preserve the original Arc when unchanged, making pointer identity sufficient. + let mut changed = !Arc::ptr_eq(&new_source, source_expr); + let mut new_children = Vec::with_capacity(children.len()); + new_children.push(new_source); + + for path_expr in path_exprs { + let field_name = path_expr + .downcast_ref::() + .and_then(|lit| lit.value().try_as_str().flatten()); + let child_field = match (¤t_field, field_name) { + (Some(source_field), Some(name)) => match source_field.data_type() { + ArrowDataType::Struct(children) => { + children.iter().find(|f| f.name() == name).cloned() + } + _ => None, + }, + _ => None, + }; + let rewritten_path = child_field.as_ref().and_then(|child| { + let physical_name = arrow_field_physical_name(child, mode); + (physical_name != child.name().as_str()).then(|| { + Arc::new(PhysicalLiteral::new(ScalarValue::Utf8(Some( + physical_name.to_string(), + )))) as Arc + }) + }); + changed |= rewritten_path.is_some(); + new_children.push(rewritten_path.unwrap_or_else(|| Arc::clone(path_expr))); + current_field = child_field; + } + + if changed { + let rewritten = Arc::clone(&expr).with_new_children(new_children)?; + return Ok((rewritten, current_field)); + } + return Ok((expr, current_field)); + } + + let children = expr.children(); + if children.is_empty() { + return Ok((expr, None)); + } + let mut new_children = Vec::with_capacity(children.len()); + let mut changed = false; + for child in children { + let (new_child, _) = rewrite_expr_for_column_mapping( + Arc::clone(child), + logical_schema, + mode, + partition_cols, + )?; + changed |= !Arc::ptr_eq(&new_child, child); + new_children.push(new_child); + } + if changed { + Ok((Arc::clone(&expr).with_new_children(new_children)?, None)) + } else { + Ok((expr, None)) + } +} + /// Extract column names referenced by a `PhysicalExpr`. pub fn collect_physical_columns(expr: &Arc) -> HashSet { let mut columns = HashSet::::new(); @@ -151,3 +272,78 @@ impl PredicateProperties { Ok(()) } } + +#[cfg(test)] +#[expect(clippy::expect_used)] +mod tests { + use std::collections::HashMap; + + use datafusion::common::config::ConfigOptions; + use datafusion::physical_expr::expressions::{ + Column as PhysicalColumn, Literal as PhysicalLiteral, + }; + + use super::*; + + fn mapped_field(name: &str, physical_name: &str, data_type: ArrowDataType) -> FieldRef { + Arc::new( + datafusion::arrow::datatypes::Field::new(name, data_type, true).with_metadata( + HashMap::from([( + "delta.columnMapping.physicalName".to_string(), + physical_name.to_string(), + )]), + ), + ) + } + + #[test] + fn rewrites_every_segment_of_flattened_get_field_path() -> Result<()> { + let inner = mapped_field("inner", "col-inner", ArrowDataType::Int64); + let outer = mapped_field( + "outer", + "col-outer", + ArrowDataType::Struct(vec![inner].into()), + ); + let root = mapped_field( + "root", + "col-root", + ArrowDataType::Struct(vec![outer].into()), + ); + let schema = ArrowSchema::new(vec![root]); + let args: Vec> = vec![ + Arc::new(PhysicalColumn::new("root", 0)), + Arc::new(PhysicalLiteral::new(ScalarValue::Utf8(Some( + "outer".to_string(), + )))), + Arc::new(PhysicalLiteral::new(ScalarValue::Utf8(Some( + "inner".to_string(), + )))), + ]; + let expr = Arc::new(ScalarFunctionExpr::try_new( + datafusion::functions::core::get_field(), + args, + &schema, + Arc::new(ConfigOptions::default()), + )?) as Arc; + + let rewritten = + rewrite_predicate_for_column_mapping(expr, &schema, ColumnMappingMode::Name, &[])?; + let children = rewritten.children(); + + let root = children[0] + .downcast_ref::() + .expect("root column"); + assert_eq!(root.name(), "col-root"); + let path = children[1..] + .iter() + .map(|child| { + child + .downcast_ref::() + .and_then(|literal| literal.value().try_as_str().flatten()) + .expect("literal path") + }) + .collect::>(); + assert_eq!(path, ["col-outer", "col-inner"]); + Ok(()) + } +} diff --git a/crates/sail-delta-lake/src/datasource/mod.rs b/crates/sail-delta-lake/src/datasource/mod.rs index b6c6c1d83..944533ce3 100644 --- a/crates/sail-delta-lake/src/datasource/mod.rs +++ b/crates/sail-delta-lake/src/datasource/mod.rs @@ -29,7 +29,7 @@ pub use sail_common_datafusion::datasource::MERGE_FILE_COLUMN as PATH_COLUMN; use serde::{Deserialize, Serialize}; use url::Url; -use crate::options::{default_delta_log_replay_hash_threshold, DeltaLogReplayStrategy}; +use crate::options::{DeltaLogReplayStrategy, default_delta_log_replay_hash_threshold}; use crate::snapshot::SnapshotPruningStats; use crate::spec::{DeltaError as DeltaTableError, DeltaResult}; use crate::table::DeltaSnapshot; @@ -45,9 +45,10 @@ pub mod schema; // Re-exports pub use actions::{adds_to_remove_actions, partitioned_file_from_action}; pub use expressions::{ - collect_physical_columns, get_pushdown_filters, simplify_expr, PredicateProperties, + PredicateProperties, collect_physical_columns, get_pushdown_filters, + rewrite_predicate_for_column_mapping, simplify_expr, }; -pub use pruning::{prune_files, PruningResult}; +pub use pruning::{PruningResult, prune_files}; pub use scan::build_file_scan_config; pub use schema::df_logical_schema; diff --git a/crates/sail-delta-lake/src/datasource/pruning.rs b/crates/sail-delta-lake/src/datasource/pruning.rs index d7ac2938e..b1d04d8d0 100644 --- a/crates/sail-delta-lake/src/datasource/pruning.rs +++ b/crates/sail-delta-lake/src/datasource/pruning.rs @@ -24,17 +24,17 @@ use datafusion::arrow::array::{ArrayRef, UInt64Array}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::Session; use datafusion::common::{Result, ToDFSchema}; -use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::Expr; +use datafusion::logical_expr::utils::conjunction; use datafusion::physical_optimizer::pruning::PruningPredicate; use datafusion_common::pruning::PruningStatistics; use datafusion_common::scalar::ScalarValue; use datafusion_common::{Column, DataFusionError}; -use crate::conversion::{parse_optional_partition_value, ScalarConverter}; +use crate::conversion::{ScalarConverter, parse_optional_partition_value}; use crate::delta_log::LogStoreRef; -use crate::spec::statistics::Stats; use crate::spec::Add; +use crate::spec::statistics::Stats; use crate::table::DeltaSnapshot; /// Result of file pruning operation @@ -92,20 +92,19 @@ pub async fn prune_files( for (action, keep) in all_files.into_iter().zip(files_to_prune.iter()) { if *keep { - if let Some(limit) = limit { - if let Some(stats) = action + if let Some(limit) = limit + && let Some(stats) = action .get_stats() .map_err(|e| datafusion::common::DataFusionError::External(Box::new(e)))? - { - if rows_collected <= limit as i64 { - rows_collected += stats.num_records; - files.push(action); - } else { - break; - } + { + if rows_collected <= limit as i64 { + rows_collected += stats.num_records; + files.push(action); } else { - pruned_without_stats.push(action); + break; } + } else if limit.is_some() { + pruned_without_stats.push(action); } else { files.push(action); } @@ -113,10 +112,10 @@ pub async fn prune_files( } // Add files without stats if we haven't reached the limit - if let Some(limit) = limit { - if rows_collected < limit as i64 { - files.extend(pruned_without_stats); - } + if let Some(limit) = limit + && rows_collected < limit as i64 + { + files.extend(pruned_without_stats); } Ok(PruningResult { @@ -398,13 +397,13 @@ impl AddStatsPruningStatistics { if let Some(pv) = a.partition_values.get(name) { return Self::scalar_from_partition_value(dt, pv); } - if let Some(s) = s { - if let Some(v) = s.min_value(name) { - return ScalarConverter::stat_value_to_arrow_scalar_value(v, dt) - .ok() - .flatten() - .unwrap_or_else(|| Self::null_scalar(dt)); - } + if let Some(s) = s + && let Some(v) = s.min_value(name) + { + return ScalarConverter::stat_value_to_arrow_scalar_value(v, dt) + .ok() + .flatten() + .unwrap_or_else(|| Self::null_scalar(dt)); } Self::null_scalar(dt) }) @@ -424,13 +423,13 @@ impl AddStatsPruningStatistics { if let Some(pv) = a.partition_values.get(name) { return Self::scalar_from_partition_value(dt, pv); } - if let Some(s) = s { - if let Some(v) = s.max_value(name) { - return ScalarConverter::stat_value_to_arrow_scalar_value(v, dt) - .ok() - .flatten() - .unwrap_or_else(|| Self::null_scalar(dt)); - } + if let Some(s) = s + && let Some(v) = s.max_value(name) + { + return ScalarConverter::stat_value_to_arrow_scalar_value(v, dt) + .ok() + .flatten() + .unwrap_or_else(|| Self::null_scalar(dt)); } Self::null_scalar(dt) }) diff --git a/crates/sail-delta-lake/src/datasource/scan.rs b/crates/sail-delta-lake/src/datasource/scan.rs index 0477a7c74..d1de0ceb3 100644 --- a/crates/sail-delta-lake/src/datasource/scan.rs +++ b/crates/sail-delta-lake/src/datasource/scan.rs @@ -22,26 +22,31 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::sync::Arc; -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, SchemaRef}; +use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field, Schema as ArrowSchema, SchemaRef, +}; use datafusion::catalog::Session; use datafusion::common::stats::{ColumnStatistics, Precision, Statistics}; use datafusion::common::{DataFusionError, Result, ScalarValue}; use datafusion::config::TableParquetOptions; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::{ - wrap_partition_type_in_dict, wrap_partition_value_in_dict, FileGroup, FileScanConfig, - FileScanConfigBuilder, ParquetSource, + FileGroup, FileScanConfig, FileScanConfigBuilder, ParquetSource, wrap_partition_type_in_dict, + wrap_partition_value_in_dict, }; use datafusion::datasource::table_schema::TableSchema; use datafusion::physical_expr::{LexOrdering, PhysicalExpr}; use object_store::path::Path; -use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapterFactory; +use sail_common_datafusion::schema_evolution::{ + SchemaEvolutionPhysicalExprAdapterFactoryWithMatching, StructFieldMatching, +}; +use sail_common_datafusion::variant::with_variant_extension_if_marked_storage; use crate::conversion::ScalarConverter; -use crate::datasource::{create_object_store_url, partitioned_file_from_action, DeltaScanConfig}; +use crate::datasource::{DeltaScanConfig, create_object_store_url, partitioned_file_from_action}; use crate::delta_log::LogStoreRef; use crate::schema::arrow_field_physical_name; -use crate::spec::{Add, MaxStat, MinStat}; +use crate::spec::{Add, ColumnMappingMode, MaxStat, MinStat}; use crate::table::DeltaSnapshot; /// Parameters for building file scan configuration @@ -108,6 +113,43 @@ pub(crate) fn file_scan_logical_names( names } +fn logical_file_schema_for_scan( + physical_file_schema: &SchemaRef, + logical_table_schema: &SchemaRef, + column_mapping_mode: ColumnMappingMode, +) -> SchemaRef { + let logical_fields_by_physical_name = logical_table_schema + .fields() + .iter() + .map(|field| { + ( + arrow_field_physical_name(field, column_mapping_mode).to_string(), + Arc::clone(field), + ) + }) + .collect::>(); + let fields = physical_file_schema + .fields() + .iter() + .map(|physical_field| { + logical_fields_by_physical_name + .get(physical_field.name()) + .map(|logical_field| { + Arc::new( + with_variant_extension_if_marked_storage(logical_field.as_ref().clone()) + .with_name(physical_field.name()), + ) + }) + .unwrap_or_else(|| Arc::clone(physical_field)) + }) + .collect::>(); + + Arc::new(ArrowSchema::new_with_metadata( + fields, + physical_file_schema.metadata().clone(), + )) +} + pub(crate) fn file_scan_projection_for_schema( snapshot: &DeltaSnapshot, scan_config: &DeltaScanConfig, @@ -151,6 +193,11 @@ pub fn build_file_scan_config( let table_partition_cols = snapshot.metadata().partition_columns(); let partition_columns_mapped = snapshot.physical_partition_columns(); let physical_to_logical = physical_to_logical_name_map(snapshot); + let logical_file_schema = logical_file_schema_for_scan( + &file_schema, + &complete_schema, + snapshot.effective_column_mapping_mode(), + ); // Build file groups by partition values let mut file_groups: HashMap< @@ -235,7 +282,8 @@ pub fn build_file_scan_config( } else { field.data_type().clone() }; - table_partition_cols_schema.push(Arc::new(Field::new(col.clone(), corrected, true))); + table_partition_cols_schema + .push(Arc::new(field.as_ref().clone().with_data_type(corrected))); } // Add file column to partition schema if configured @@ -272,7 +320,7 @@ pub fn build_file_scan_config( ..Default::default() }; - let table_schema = TableSchema::new(Arc::clone(&file_schema), table_partition_cols_schema); + let table_schema = TableSchema::new(logical_file_schema, table_partition_cols_schema); // Calculate table statistics. // // `Statistics::column_statistics` expects the same length as the table schema @@ -330,10 +378,10 @@ pub fn build_file_scan_config( let mut parquet_source = ParquetSource::new(table_schema).with_table_parquet_options(parquet_options); - if let Some(predicate) = params.pushdown_filter { - if config.enable_parquet_pushdown { - parquet_source = parquet_source.with_predicate(predicate); - } + if let Some(predicate) = params.pushdown_filter + && config.enable_parquet_pushdown + { + parquet_source = parquet_source.with_predicate(predicate); } let file_source: Arc = @@ -364,7 +412,15 @@ pub fn build_file_scan_config( .with_statistics(stats) .with_projection_indices(params.projection.cloned())? .with_limit(params.limit) - .with_expr_adapter(Some(Arc::new(SchemaEvolutionPhysicalExprAdapterFactory {}))) + .with_expr_adapter(Some(Arc::new( + SchemaEvolutionPhysicalExprAdapterFactoryWithMatching::new_relaxed_timezone( + match snapshot.effective_column_mapping_mode() { + crate::spec::ColumnMappingMode::None => StructFieldMatching::Name, + crate::spec::ColumnMappingMode::Name => StructFieldMatching::PhysicalName, + crate::spec::ColumnMappingMode::Id => StructFieldMatching::FieldId, + }, + ), + ))) .build(); Ok(file_scan_config) @@ -433,21 +489,20 @@ fn sanitize_column_statistics_for_field( .max_value .get_value() .map(ScalarValue::data_type); - if let (Some(min_type), Some(max_type)) = (min_type, max_type) { - if min_type != max_type { - column_stats.min_value = Precision::Absent; - column_stats.max_value = Precision::Absent; - return; - } + if let (Some(min_type), Some(max_type)) = (min_type, max_type) + && min_type != max_type + { + column_stats.min_value = Precision::Absent; + column_stats.max_value = Precision::Absent; + return; } if let (Some(min), Some(max)) = ( column_stats.min_value.get_value(), column_stats.max_value.get_value(), - ) { - if matches!(min.partial_cmp(max), Some(Ordering::Greater)) { - column_stats.min_value = Precision::Absent; - column_stats.max_value = Precision::Absent; - } + ) && matches!(min.partial_cmp(max), Some(Ordering::Greater)) + { + column_stats.min_value = Precision::Absent; + column_stats.max_value = Precision::Absent; } } @@ -609,14 +664,13 @@ fn stats_for_add( ScalarConverter::stat_value_to_arrow_scalar_value(v, field.data_type()) .ok() .flatten() - }) { - if !value.is_null() { - min_value = match min_stat { - MinStat::Exact(_) => Precision::Exact(value), - MinStat::LowerBound(_) => Precision::Inexact(value), - MinStat::Absent => Precision::Absent, - }; - } + }) && !value.is_null() + { + min_value = match min_stat { + MinStat::Exact(_) => Precision::Exact(value), + MinStat::LowerBound(_) => Precision::Inexact(value), + MinStat::Absent => Precision::Absent, + }; } } if max_value == Precision::Absent { @@ -625,25 +679,24 @@ fn stats_for_add( ScalarConverter::stat_value_to_arrow_scalar_value(v, field.data_type()) .ok() .flatten() - }) { - if !value.is_null() { - max_value = match max_stat { - MaxStat::Exact(_) => Precision::Exact(value), - MaxStat::UpperBound(_) => Precision::Inexact(value), - MaxStat::Absent => Precision::Absent, - }; - } - } - } - if null_count == Precision::Absent { - if let Some(value) = stats.null_count_value(name) { - null_count = if stats.tight_bounds { - Precision::Exact(value.max(0) as usize) - } else { - Precision::Inexact(value.max(0) as usize) + }) && !value.is_null() + { + max_value = match max_stat { + MaxStat::Exact(_) => Precision::Exact(value), + MaxStat::UpperBound(_) => Precision::Inexact(value), + MaxStat::Absent => Precision::Absent, }; } } + if null_count == Precision::Absent + && let Some(value) = stats.null_count_value(name) + { + null_count = if stats.tight_bounds { + Precision::Exact(value.max(0) as usize) + } else { + Precision::Inexact(value.max(0) as usize) + }; + } } column_statistics.push(ColumnStatistics { @@ -675,8 +728,8 @@ mod tests { use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::common::stats::{ColumnStatistics, Precision, Statistics}; use datafusion::common::ScalarValue; + use datafusion::common::stats::{ColumnStatistics, Precision, Statistics}; use object_store::path::Path; use super::{ diff --git a/crates/sail-delta-lake/src/deletion_vector/mod.rs b/crates/sail-delta-lake/src/deletion_vector/mod.rs index a0b286fff..584e0182d 100644 --- a/crates/sail-delta-lake/src/deletion_vector/mod.rs +++ b/crates/sail-delta-lake/src/deletion_vector/mod.rs @@ -7,4 +7,4 @@ pub(crate) mod z85; pub use bitmap::DeletionVectorBitmap; pub use resolve::resolve_dv_absolute_path; -pub use storage::{read_deletion_vector, write_deletion_vector, DeletionVectorWriter}; +pub use storage::{DeletionVectorWriter, read_deletion_vector, write_deletion_vector}; diff --git a/crates/sail-delta-lake/src/deletion_vector/z85.rs b/crates/sail-delta-lake/src/deletion_vector/z85.rs index 8262e6be9..450e5c046 100644 --- a/crates/sail-delta-lake/src/deletion_vector/z85.rs +++ b/crates/sail-delta-lake/src/deletion_vector/z85.rs @@ -15,8 +15,10 @@ pub(crate) fn z85_encode(data: &[u8]) -> DeltaResult { ))); } let mut result = String::with_capacity(data.len() * 5 / 4); - for chunk in data.chunks(4) { - let mut value = u32::from_be_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) as u64; + let (chunks, remainder) = data.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + for chunk in chunks { + let mut value = u32::from_be_bytes(*chunk) as u64; let mut chars = [0u8; 5]; for c in chars.iter_mut().rev() { *c = Z85_CHARS[(value % 85) as usize]; @@ -57,7 +59,9 @@ pub(crate) fn z85_decode(encoded: &str) -> DeltaResult> { } let mut result = Vec::with_capacity(bytes.len() * 4 / 5); - for chunk in bytes.chunks(5) { + let (chunks, remainder) = bytes.as_chunks::<5>(); + debug_assert!(remainder.is_empty()); + for chunk in chunks { let mut value: u64 = 0; for &b in chunk { let idx = z85_char_to_value(b)?; diff --git a/crates/sail-delta-lake/src/delta_log/cleanup.rs b/crates/sail-delta-lake/src/delta_log/cleanup.rs index ddcc70e5d..1b3feadeb 100644 --- a/crates/sail-delta-lake/src/delta_log/cleanup.rs +++ b/crates/sail-delta-lake/src/delta_log/cleanup.rs @@ -16,8 +16,8 @@ use crate::checkpoint::read_checkpoint_main_rows_from_checkpoint_file; use crate::delta_log::LogStore; use crate::snapshot::DeltaSnapshot; use crate::spec::{ - checkpoint_path, delta_log_root_path, is_uuid_checkpoint_filename, sidecar_file_name, - sidecars_dir_path, DeltaResult, + DeltaResult, checkpoint_path, delta_log_root_path, is_uuid_checkpoint_filename, + sidecar_file_name, sidecars_dir_path, }; #[derive(Debug, Clone, Copy)] @@ -174,13 +174,13 @@ async fn ensure_v2_compat_classic_checkpoint( .rsplit('/') .next() .unwrap_or_default(); - if is_uuid_checkpoint_filename(filename) && filename.ends_with(".parquet") { - if let Some(v) = parse_checkpoint_version_from_location(&meta.location) { - if v == version { - uuid_checkpoint_meta = Some(meta); - break; - } - } + if is_uuid_checkpoint_filename(filename) + && filename.ends_with(".parquet") + && let Some(v) = parse_checkpoint_version_from_location(&meta.location) + && v == version + { + uuid_checkpoint_meta = Some(meta); + break; } } @@ -378,11 +378,11 @@ mod tests { use url::Url; use super::*; - use crate::delta_log::{default_logstore, LogStoreRef, StorageConfig}; + use crate::delta_log::{LogStoreRef, StorageConfig, default_logstore}; use crate::snapshot::DeltaSnapshot; use crate::spec::{ - checkpoint_path, checksum_path, commit_path, Action, CommitInfo, DataType, Metadata, - Protocol, StructField, StructType, TableFeature, VersionChecksum, + Action, CommitInfo, DataType, Metadata, Protocol, StructField, StructType, TableFeature, + VersionChecksum, checkpoint_path, checksum_path, commit_path, }; fn test_log_store(store: Arc) -> LogStoreRef { diff --git a/crates/sail-delta-lake/src/delta_log/listing.rs b/crates/sail-delta-lake/src/delta_log/listing.rs index e0024fd36..89847b910 100644 --- a/crates/sail-delta-lake/src/delta_log/listing.rs +++ b/crates/sail-delta-lake/src/delta_log/listing.rs @@ -5,9 +5,9 @@ use object_store::path::Path; use object_store::{Error as ObjectStoreError, ObjectMeta, ObjectStore, ObjectStoreExt}; use crate::spec::{ - delta_log_prefix_path, delta_log_root_path, last_checkpoint_path, parse_checkpoint_version, - parse_checksum_version, parse_commit_version, parse_compacted_json_versions, DeltaResult, - LastCheckpointHint, + DeltaResult, LastCheckpointHint, delta_log_prefix_path, delta_log_root_path, + last_checkpoint_path, parse_checkpoint_version, parse_checksum_version, parse_commit_version, + parse_compacted_json_versions, }; pub(crate) fn parse_delta_log_entry_version(meta: &ObjectMeta) -> Option { diff --git a/crates/sail-delta-lake/src/delta_log/mod.rs b/crates/sail-delta-lake/src/delta_log/mod.rs index 8c1c0f2a9..1a93a7b25 100644 --- a/crates/sail-delta-lake/src/delta_log/mod.rs +++ b/crates/sail-delta-lake/src/delta_log/mod.rs @@ -16,12 +16,12 @@ pub(crate) use replay::{ latest_replayable_version, load_replayed_table_header, load_replayed_table_state, }; pub(crate) use segment::{ - list_log_files, LogSegmentResolver, ReplayedTableHeader, ResolvedLogSegment, + LogSegmentResolver, ReplayedTableHeader, ResolvedLogSegment, list_log_files, }; -pub(crate) use store::{default_logstore, get_actions, get_object_store_from_context}; pub use store::{ CommitOrBytes, LogStore, LogStoreConfig, LogStoreRef, ObjectStoreRef, StorageConfig, }; +pub(crate) use store::{default_logstore, get_actions, get_object_store_from_context}; pub(crate) use timestamps::{ resolve_commit_timestamp_from_actions, resolve_effective_protocol_and_metadata, resolve_version_timestamp, diff --git a/crates/sail-delta-lake/src/delta_log/replay.rs b/crates/sail-delta-lake/src/delta_log/replay.rs index 25b88af73..9de6bbeaa 100644 --- a/crates/sail-delta-lake/src/delta_log/replay.rs +++ b/crates/sail-delta-lake/src/delta_log/replay.rs @@ -4,25 +4,23 @@ use std::sync::Arc; use datafusion::common::runtime::SpawnedTask; use log::debug; use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; -use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use super::{ - list_delta_log_entries_from, parse_checkpoint_version_from_location, - parse_commit_version_from_location, parse_compacted_json_versions_from_location, - read_last_checkpoint_version_from_store, LogSegmentResolver, ReplayedTableHeader, - ResolvedLogSegment, + LogSegmentResolver, ReplayedTableHeader, ResolvedLogSegment, list_delta_log_entries_from, + parse_checkpoint_version_from_location, parse_commit_version_from_location, + parse_compacted_json_versions_from_location, read_last_checkpoint_version_from_store, }; use crate::checkpoint::{ - decode_checkpoint_rows, read_checkpoint_main_rows_from_checkpoint_file, - read_checkpoint_rows_from_checkpoint_file, replay_commit_actions_with_compactions, - replay_commit_header_actions_with_compactions, ReconciledCheckpointState, - ReconciledHeaderState, ReplayedTableState, + ReconciledCheckpointState, ReconciledHeaderState, ReplayedTableState, decode_checkpoint_rows, + read_checkpoint_main_rows_from_checkpoint_file, read_checkpoint_rows_from_checkpoint_file, + replay_commit_actions_with_compactions, replay_commit_header_actions_with_compactions, }; use crate::delta_log::LogStore; use crate::snapshot::CatalogManagedCommitSet; use crate::spec::{ - is_json_checkpoint_filename, CheckpointActionRow, DeltaError as DeltaTableError, DeltaResult, + CheckpointActionRow, DeltaError as DeltaTableError, DeltaResult, is_json_checkpoint_filename, }; async fn read_checkpoint_header_from_checkpoint_file( diff --git a/crates/sail-delta-lake/src/delta_log/segment.rs b/crates/sail-delta-lake/src/delta_log/segment.rs index 52fccc9ec..a275ce838 100644 --- a/crates/sail-delta-lake/src/delta_log/segment.rs +++ b/crates/sail-delta-lake/src/delta_log/segment.rs @@ -7,11 +7,11 @@ use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; use super::timestamps::version_uses_in_commit_timestamps; use super::{list_delta_log_entries_from, read_last_checkpoint_version_from_store}; use crate::delta_log::LogStore; -use crate::snapshot::{catalog_managed_commit_path, CatalogManagedCommitSet}; +use crate::snapshot::{CatalogManagedCommitSet, catalog_managed_commit_path}; use crate::spec::{ + DeltaError, DeltaResult, DomainMetadata, Metadata, Protocol, Transaction, VersionChecksum, checksum_path, parse_checkpoint_version, parse_checksum_version, parse_commit_version, - parse_compacted_json_versions, DeltaError, DeltaResult, DomainMetadata, Metadata, Protocol, - Transaction, VersionChecksum, + parse_compacted_json_versions, }; const CHECKSUM_LOOKBACK_WINDOW: i64 = 100; @@ -113,11 +113,11 @@ impl<'a> LogSegmentResolver<'a> { let version = self.target_version; let store = self.log_store.object_store(None); - if !self.has_catalog_managed_commits_in_range(0, version) { - if let Some(header) = try_read_checksum_header(store.clone(), version).await { - debug!("crc-header: exact checksum hit target_version={version}"); - return Ok(ResolvedLogSegment::ExactChecksum { header }); - } + if !self.has_catalog_managed_commits_in_range(0, version) + && let Some(header) = try_read_checksum_header(store.clone(), version).await + { + debug!("crc-header: exact checksum hit target_version={version}"); + return Ok(ResolvedLogSegment::ExactChecksum { header }); } let lower_bound = { @@ -417,22 +417,22 @@ pub(crate) async fn list_log_files( Some(f) => f, None => continue, }; - if let Some(v) = parse_checkpoint_version(filename) { - if v <= max_version { - checkpoint_candidates.push((v, meta)); - } + if let Some(v) = parse_checkpoint_version(filename) + && v <= max_version + { + checkpoint_candidates.push((v, meta)); continue; } - if let Some(v) = parse_commit_version(filename) { - if v <= max_version { - commit_candidates.push((v, meta)); - } + if let Some(v) = parse_commit_version(filename) + && v <= max_version + { + commit_candidates.push((v, meta)); continue; } - if let Some(v) = parse_checksum_version(filename) { - if v <= max_version { - checksum_candidates.push((v, meta)); - } + if let Some(v) = parse_checksum_version(filename) + && v <= max_version + { + checksum_candidates.push((v, meta)); continue; } if let Some(versions) = parse_compacted_json_versions(filename) { @@ -559,10 +559,10 @@ fn resolve_compactions( for ((start, end), meta) in compaction_files { // Skip if this compaction overlaps with an already-selected one. - if let Some(boundary) = covered_up_to { - if end >= boundary { - continue; - } + if let Some(boundary) = covered_up_to + && end >= boundary + { + continue; } selected_compactions.push(((start, end), meta)); covered_up_to = Some(start); diff --git a/crates/sail-delta-lake/src/delta_log/segment_files.rs b/crates/sail-delta-lake/src/delta_log/segment_files.rs index c88e802b3..a9934f78f 100644 --- a/crates/sail-delta-lake/src/delta_log/segment_files.rs +++ b/crates/sail-delta-lake/src/delta_log/segment_files.rs @@ -13,7 +13,7 @@ use crate::checkpoint::read_checkpoint_main_rows_from_checkpoint_file; pub(crate) use crate::delta_log::ReplayedTableHeader; use crate::delta_log::{list_log_files, read_last_checkpoint_version_from_store}; -use crate::spec::{is_uuid_checkpoint_filename, sidecar_log_path, DeltaResult}; +use crate::spec::{DeltaResult, is_uuid_checkpoint_filename, sidecar_log_path}; /// The minimal set of Delta log files needed to reconstruct table state up to a given version. #[derive(Debug, Clone, Default)] diff --git a/crates/sail-delta-lake/src/delta_log/store.rs b/crates/sail-delta-lake/src/delta_log/store.rs index f4dd14d85..2c5963d0f 100644 --- a/crates/sail-delta-lake/src/delta_log/store.rs +++ b/crates/sail-delta-lake/src/delta_log/store.rs @@ -35,7 +35,7 @@ use uuid::Uuid; use crate::delta_log::latest_version_from_listing; use crate::spec::{ - commit_path, Action, DeltaError as DeltaTableError, DeltaError, DeltaResult, TransactionError, + Action, DeltaError as DeltaTableError, DeltaError, DeltaResult, TransactionError, commit_path, }; mod config; diff --git a/crates/sail-delta-lake/src/delta_log/store/config.rs b/crates/sail-delta-lake/src/delta_log/store/config.rs index f2bd833c8..7a884769e 100644 --- a/crates/sail-delta-lake/src/delta_log/store/config.rs +++ b/crates/sail-delta-lake/src/delta_log/store/config.rs @@ -18,9 +18,9 @@ use std::sync::Arc; +use object_store::ObjectStore; use object_store::path::Path; use object_store::prefix::PrefixStore; -use object_store::ObjectStore; use url::Url; use crate::spec::DeltaResult; diff --git a/crates/sail-delta-lake/src/delta_log/timestamps.rs b/crates/sail-delta-lake/src/delta_log/timestamps.rs index d2fb68ed6..cb38b2f93 100644 --- a/crates/sail-delta-lake/src/delta_log/timestamps.rs +++ b/crates/sail-delta-lake/src/delta_log/timestamps.rs @@ -1,9 +1,9 @@ use object_store::{ObjectMeta, ObjectStoreExt}; -use crate::delta_log::{get_actions, LogStore}; +use crate::delta_log::{LogStore, get_actions}; use crate::spec::{ - checksum_path, commit_path, Action, DeltaError, DeltaResult, Metadata, Protocol, - TableProperties, VersionChecksum, + Action, DeltaError, DeltaResult, Metadata, Protocol, TableProperties, VersionChecksum, + checksum_path, commit_path, }; pub(crate) fn in_commit_timestamp_from_actions(actions: &[Action]) -> Option { @@ -146,7 +146,7 @@ mod tests { use url::Url; use super::*; - use crate::delta_log::{default_logstore, LogStoreRef, StorageConfig}; + use crate::delta_log::{LogStoreRef, StorageConfig, default_logstore}; use crate::spec::{CommitInfo, DataType, Metadata, StructField, StructType, TableFeature}; fn test_log_store(store: Arc) -> LogStoreRef { @@ -253,8 +253,8 @@ mod tests { } #[tokio::test] - async fn resolve_version_timestamp_ignores_pre_enable_ict_in_checksum_and_commit( - ) -> DeltaResult<()> { + async fn resolve_version_timestamp_ignores_pre_enable_ict_in_checksum_and_commit() + -> DeltaResult<()> { let store: Arc = Arc::new(InMemory::new()); let log_store = test_log_store(store.clone()); let pre_enable_protocol = Protocol::new(1, 2, None, None); diff --git a/crates/sail-delta-lake/src/logical/merge.rs b/crates/sail-delta-lake/src/logical/merge.rs index 188e5c5b4..911425e00 100644 --- a/crates/sail-delta-lake/src/logical/merge.rs +++ b/crates/sail-delta-lake/src/logical/merge.rs @@ -7,15 +7,16 @@ use datafusion_expr::logical_plan::Extension; use datafusion_expr::{Expr, LogicalPlan, TableScan, TableSource}; use log::trace; use sail_common_datafusion::datasource::{ - MergeCapableSource, MergeInfo, MergeMatchedAction, MergeNotMatchedBySourceAction, - MERGE_FILE_COLUMN, MERGE_ROW_INDEX_COLUMN, + MERGE_FILE_COLUMN, MERGE_ROW_INDEX_COLUMN, MergeCapableSource, MergeInfo, MergeMatchedAction, + MergeNotMatchedBySourceAction, }; -use sail_logical_plan::merge::{expand_merge, RowLevelWriteNode}; +use sail_logical_plan::merge::{RowLevelWriteNode, expand_merge, validate_merge_internal_columns}; use crate::logical::table_source::DeltaTableSource; /// Expand MERGE information into a unified row-level write node for Delta. pub fn expand_merge_node(info: MergeInfo) -> Result { + validate_merge_internal_columns(&info, &[MERGE_FILE_COLUMN, MERGE_ROW_INDEX_COLUMN])?; let row_index_column = (merge_has_delete_actions(&info) && merge_target_supports_deletion_vectors(info.target.as_ref())?) .then_some(MERGE_ROW_INDEX_COLUMN); @@ -32,7 +33,7 @@ pub fn expand_merge_node(info: MergeInfo) -> Result { .collect(); trace!( "rewrite target_plan schema after ensure_merge_metadata_columns: {:?}", - &target_fields + target_fields ); if !target_fields.iter().any(|n| n == MERGE_FILE_COLUMN) || row_index_column.is_some_and(|c| !target_fields.iter().any(|n| n == c)) @@ -44,12 +45,10 @@ pub fn expand_merge_node(info: MergeInfo) -> Result { if !target_fields.iter().any(|n| n == MERGE_FILE_COLUMN) { exprs.push(Expr::Column(Column::from_name(MERGE_FILE_COLUMN)).alias(MERGE_FILE_COLUMN)); } - if let Some(row_index_column) = row_index_column { - if !target_fields.iter().any(|n| n == row_index_column) { - exprs.push( - Expr::Column(Column::from_name(row_index_column)).alias(row_index_column), - ); - } + if let Some(row_index_column) = row_index_column + && !target_fields.iter().any(|n| n == row_index_column) + { + exprs.push(Expr::Column(Column::from_name(row_index_column)).alias(row_index_column)); } target_plan = LogicalPlanBuilder::from(target_plan) .project(exprs)? @@ -74,7 +73,7 @@ pub fn expand_merge_node(info: MergeInfo) -> Result { let raw_target = Arc::clone(&info.target); let raw_source = Arc::clone(&info.source); let raw_input_schema = info.input_schema.clone(); - let expansion = expand_merge(info, MERGE_FILE_COLUMN, row_index_column)?; + let expansion = expand_merge(info, MERGE_FILE_COLUMN, row_index_column, &[])?; trace!( "MERGE expansion write_plan schema fields: {:?}", expansion @@ -91,7 +90,7 @@ pub fn expand_merge_node(info: MergeInfo) -> Result { raw_input_schema, Arc::new(expansion.write_plan), Arc::new(expansion.touched_files_plan), - expansion.deletion_vector_plan.map(Arc::new), + expansion.row_index_delete_plan.map(Arc::new), expansion.options, expansion.output_schema, ); @@ -116,11 +115,11 @@ fn merge_has_delete_actions(info: &MergeInfo) -> bool { fn merge_target_supports_deletion_vectors(plan: &LogicalPlan) -> Result { let mut supports = false; plan.apply(|node| { - if let LogicalPlan::TableScan(scan) = node { - if let Some(delta_source) = scan.source.downcast_ref::() { - supports = delta_source.snapshot().verify_deletion_vectors().is_ok(); - return Ok(TreeNodeRecursion::Stop); - } + if let LogicalPlan::TableScan(scan) = node + && let Some(delta_source) = scan.source.downcast_ref::() + { + supports = delta_source.snapshot().verify_deletion_vectors().is_ok(); + return Ok(TreeNodeRecursion::Stop); } Ok(TreeNodeRecursion::Continue) })?; @@ -151,11 +150,10 @@ fn try_enable_merge_metadata_columns( } if let (Some(row_index_col), Some(delta_source)) = (row_index_col, new_source.downcast_ref::()) + && delta_source.row_index_column_name().is_none() { - if delta_source.row_index_column_name().is_none() { - new_source = delta_source.with_row_index_column(row_index_col)?; - changed = true; - } + new_source = delta_source.with_row_index_column(row_index_col)?; + changed = true; } if changed { let schema = new_source.schema(); @@ -178,14 +176,14 @@ fn ensure_merge_metadata_columns( let transformed = plan .transform_up(|plan| { // First, configure table scans to expose the file path column. - if let LogicalPlan::TableScan(scan) = &plan { - if let Some((new_source, schema)) = + if let LogicalPlan::TableScan(scan) = &plan + && let Some((new_source, schema)) = try_enable_merge_metadata_columns(&scan.source, file_col, row_index_col)? { trace!( "ensure_merge_metadata_columns (scan) before - table_name: {:?}, projection: {:?}", - &scan.table_name, - &scan.projection + scan.table_name, + scan.projection ); let mut projection: Option> = scan.projection.clone(); @@ -194,11 +192,10 @@ fn ensure_merge_metadata_columns( } if let Some(proj) = projection.as_mut() { for col in &metadata_cols { - if let Some(idx) = schema.column_with_name(col).map(|(idx, _)| idx) { - if !proj.contains(&idx) { + if let Some(idx) = schema.column_with_name(col).map(|(idx, _)| idx) + && !proj.contains(&idx) { proj.push(idx); } - } } } @@ -221,7 +218,6 @@ fn ensure_merge_metadata_columns( return Ok(Transformed::yes(new_scan)); } - } // Then ensure parent projections keep metadata columns if present in input. if let LogicalPlan::Projection(proj) = &plan { diff --git a/crates/sail-delta-lake/src/logical/table_source.rs b/crates/sail-delta-lake/src/logical/table_source.rs index ba1ba8df1..16c36c65f 100644 --- a/crates/sail-delta-lake/src/logical/table_source.rs +++ b/crates/sail-delta-lake/src/logical/table_source.rs @@ -5,10 +5,10 @@ use datafusion::common::Result; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource}; use sail_common_datafusion::datasource::MergeCapableSource; -use crate::datasource::{df_logical_schema, get_pushdown_filters, DeltaScanConfig}; +use crate::DeltaResult; +use crate::datasource::{DeltaScanConfig, df_logical_schema, get_pushdown_filters}; use crate::delta_log::LogStoreRef; use crate::table::DeltaSnapshot; -use crate::DeltaResult; /// Logical-only Delta table source used in DataFusion logical plans. /// diff --git a/crates/sail-delta-lake/src/options.rs b/crates/sail-delta-lake/src/options.rs index a73fa1299..2ad40ff1c 100644 --- a/crates/sail-delta-lake/src/options.rs +++ b/crates/sail-delta-lake/src/options.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use crate::error::{DataSourceError, DataSourceResult}; -pub mod gen { +pub mod r#gen { include!(concat!(env!("OUT_DIR"), "/options/delta.rs")); } @@ -65,9 +65,9 @@ fn non_zero_usize(value: usize) -> NonZeroUsize { } } -impl ResolveOptions for gen::DeltaReadOptions { +impl ResolveOptions for r#gen::DeltaReadOptions { fn resolve(_ctx: &dyn Session, options: Vec) -> DataSourceResult { - let mut partial = gen::DeltaReadPartialOptions::initialize(); + let mut partial = r#gen::DeltaReadPartialOptions::initialize(); for layer in options { partial.merge(layer.build_partial_options()?); } @@ -75,9 +75,9 @@ impl ResolveOptions for gen::DeltaReadOptions { } } -impl ResolveOptions for gen::DeltaWriteOptions { +impl ResolveOptions for r#gen::DeltaWriteOptions { fn resolve(_ctx: &dyn Session, options: Vec) -> DataSourceResult { - let mut partial = gen::DeltaWritePartialOptions::initialize(); + let mut partial = r#gen::DeltaWritePartialOptions::initialize(); for layer in options { partial.merge(layer.build_partial_options()?); } diff --git a/crates/sail-delta-lake/src/physical/scan_planner.rs b/crates/sail-delta-lake/src/physical/scan_planner.rs index 3b5ab78f1..44b5ead69 100644 --- a/crates/sail-delta-lake/src/physical/scan_planner.rs +++ b/crates/sail-delta-lake/src/physical/scan_planner.rs @@ -1,33 +1,34 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use datafusion::arrow::datatypes::Schema as ArrowSchema; +use datafusion::arrow::datatypes::{DataType as ArrowDataType, Schema as ArrowSchema, SchemaRef}; use datafusion::catalog::Session; -use datafusion::common::{Result, ToDFSchema}; +use datafusion::common::{DataFusionError, Result, ToDFSchema}; use datafusion::datasource::source::DataSourceExec; -use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::Expr; -use datafusion::physical_expr::expressions::Column; +use datafusion::logical_expr::utils::conjunction; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::expressions::{CastExpr, Column}; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::{ExecutionPlan, Partitioning}; -use sail_common_datafusion::rename::physical_plan::rename_physical_plan; use sail_data_source::options::ResolveOptions; use crate::datasource::scan::{ - build_file_scan_config, file_scan_projection_for_schema, FileScanParams, TableStatsMode, + FileScanParams, TableStatsMode, build_file_scan_config, file_scan_projection_for_schema, +}; +use crate::datasource::{ + DeltaScanConfig, df_logical_schema, rewrite_predicate_for_column_mapping, simplify_expr, }; -use crate::datasource::{df_logical_schema, simplify_expr, DeltaScanConfig}; use crate::delta_log::LogStoreRef; -use crate::options::gen::DeltaWriteOptions; +use crate::options::r#gen::DeltaWriteOptions; use crate::physical_plan::planner::metadata_predicate::{ build_metadata_filter, predicate_requires_stats, }; use crate::physical_plan::planner::utils::LogReplayOptions; use crate::physical_plan::planner::{DeltaPlannerConfig, PlannerContext}; -use crate::physical_plan::{DeltaDiscoveryExec, DeltaScanByAddsExec, RelaxedTzCastExec}; -use crate::schema::get_physical_schema; +use crate::physical_plan::{DeltaDiscoveryExec, DeltaScanByAddsExec}; +use crate::schema::{attach_column_mapping_metadata, get_physical_schema}; use crate::spec::{Add, ColumnMappingMode, StructType}; use crate::table::DeltaSnapshot; @@ -41,15 +42,24 @@ pub(crate) async fn plan_delta_scan( filters: &[Expr], limit: Option, ) -> Result> { - let config = config.clone(); + let mut config = config.clone(); snapshot .ensure_data_read_supported() .map_err(|e| datafusion::common::DataFusionError::External(Box::new(e)))?; - let schema = config - .schema - .clone() - .unwrap_or_else(|| Arc::new(snapshot.schema().clone())); + let kmode = snapshot.effective_column_mapping_mode(); + let schema = match config.schema.clone() { + Some(requested) if kmode != ColumnMappingMode::None => { + let schema = Arc::new(attach_column_mapping_metadata( + requested.as_ref(), + snapshot.schema(), + )); + config.schema = Some(Arc::clone(&schema)); + schema + } + Some(requested) => requested, + None => Arc::new(snapshot.schema().clone()), + }; let full_logical_schema = df_logical_schema( snapshot, @@ -78,11 +88,11 @@ pub(crate) async fn plan_delta_scan( } // Ensure all partition columns are included in logical schema for partition_col in table_partition_cols.iter() { - if let Ok(idx) = full_logical_schema.index_of(partition_col.as_str()) { - if !used_columns.contains(&idx) && !fields.iter().any(|f| f.name() == partition_col) - { - fields.push(full_logical_schema.field(idx).to_owned()); - } + if let Ok(idx) = full_logical_schema.index_of(partition_col.as_str()) + && !used_columns.contains(&idx) + && !fields.iter().any(|f| f.name() == partition_col) + { + fields.push(full_logical_schema.field(idx).to_owned()); } } Arc::new(ArrowSchema::new(fields)) @@ -102,10 +112,10 @@ pub(crate) async fn plan_delta_scan( } } for partition_col in table_partition_cols.iter() { - if let Ok(idx) = full_logical_schema.index_of(partition_col.as_str()) { - if !scan_projection.contains(&idx) { - scan_projection.push(idx); - } + if let Ok(idx) = full_logical_schema.index_of(partition_col.as_str()) + && !scan_projection.contains(&idx) + { + scan_projection.push(idx); } } (Some(scan_projection), Some(used_columns.len())) @@ -175,10 +185,9 @@ pub(crate) async fn plan_delta_scan( }; // Build physical file schema (non-partition columns) - let kmode: ColumnMappingMode = snapshot.effective_column_mapping_mode(); let kschema_arc = snapshot.schema(); let logical_kernel = StructType::try_from(kschema_arc)?; - let physical_arrow: ArrowSchema = get_physical_schema(&logical_kernel, kmode); + let physical_arrow: ArrowSchema = get_physical_schema(&logical_kernel, kmode)?; let physical_partition_cols: HashSet = snapshot .physical_partition_columns() .into_iter() @@ -209,6 +218,18 @@ pub(crate) async fn plan_delta_scan( } else { None }; + // The parquet scan resolves predicate columns against the physical file schema, + // so column-mapped tables need the predicate rewritten from logical to physical names. + let pushdown_filter = pushdown_filter + .map(|expr| { + rewrite_predicate_for_column_mapping( + expr, + snapshot.schema(), + kmode, + &table_partition_cols, + ) + }) + .transpose()?; // When the table protocol declares the deletionVectors feature, always use the // metadata-as-data path (DeltaScanByAddsExec) which loads a fresh snapshot and @@ -258,29 +279,7 @@ pub(crate) async fn plan_delta_scan( )?; let scan_exec = DataSourceExec::from_data_source(file_scan_config); - - // Rename columns from physical back to logical names expected by the Spark-facing schema. - let logical_names = output_schema - .fields() - .iter() - .map(|f| f.name().clone()) - .collect::>(); - let renamed = rename_physical_plan(scan_exec, &logical_names)?; - - let renamed_schema = renamed.schema(); - - let needs_wrapping = output_schema.fields().iter().any(|field| { - let Ok(input_field) = renamed_schema.field_with_name(field.name()) else { - return true; - }; - input_field.data_type() != field.data_type() - }); - if needs_wrapping { - return Ok( - Arc::new(RelaxedTzCastExec::new(renamed, output_schema)) as Arc - ); - } - return Ok(renamed); + return align_delta_scan_output(scan_exec, output_schema); } // Metadata-as-data path: log scan -> replay -> discovery -> scan by adds. @@ -367,6 +366,7 @@ pub(crate) async fn plan_delta_scan( limit, pushdown_filter, None, + snapshot.load_config().catalog_managed_commits.clone(), ) .with_table_statistics(snapshot.datafusion_table_statistics(None)), ); @@ -388,3 +388,120 @@ pub(crate) async fn plan_delta_scan( Ok(scan_exec) } + +fn align_delta_scan_output( + input: Arc, + target_schema: SchemaRef, +) -> Result> { + let input_schema = input.schema(); + if input_schema.fields().len() != target_schema.fields().len() { + return Err(DataFusionError::Plan(format!( + "cannot align Delta scan with {} fields to logical schema with {} fields", + input_schema.fields().len(), + target_schema.fields().len() + ))); + } + if input_schema == target_schema { + return Ok(input); + } + + let expressions = input_schema + .fields() + .iter() + .zip(target_schema.fields()) + .enumerate() + .map(|(index, (input_field, target_field))| { + let column = Arc::new(Column::new(input_field.name(), index)) as Arc; + let renamed_input_field = input_field.as_ref().clone().with_name(target_field.name()); + let expression: Arc = if &renamed_input_field == target_field.as_ref() + { + column + } else { + if input_field.data_type() != target_field.data_type() + && matches!( + (input_field.data_type(), target_field.data_type()), + ( + ArrowDataType::Timestamp(_, _), + ArrowDataType::Timestamp(_, _) + ) + ) + { + return Err(DataFusionError::Plan(format!( + "Delta Parquet scan did not restore timestamp field '{}' from {} to {}", + target_field.name(), + input_field.data_type(), + target_field.data_type() + ))); + } + Arc::new(CastExpr::new_with_target_field( + column, + Arc::clone(target_field), + None, + )) + }; + Ok((expression, target_field.name().clone())) + }) + .collect::>>()?; + let projection = + Arc::new(ProjectionExec::try_new(expressions, input)?) as Arc; + + if projection.schema() != target_schema { + return Err(DataFusionError::Plan(format!( + "Delta scan projection produced schema {} instead of {}", + projection.schema(), + target_schema + ))); + } + Ok(projection) +} + +#[cfg(test)] +mod tests { + use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; + use datafusion::physical_plan::empty::EmptyExec; + + use super::*; + + #[test] + fn align_delta_scan_output_reuses_exact_input() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, true)])); + let input = Arc::new(EmptyExec::new(Arc::clone(&schema))) as Arc; + + let aligned = align_delta_scan_output(Arc::clone(&input), schema)?; + + assert!(Arc::ptr_eq(&aligned, &input)); + Ok(()) + } + + #[test] + fn align_delta_scan_output_rejects_unrestored_timestamp_timezone() -> Result<()> { + let input_schema = Arc::new(Schema::new(vec![Field::new( + "event_time", + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))), + true, + )])); + let target_schema = Arc::new(Schema::new(vec![Field::new( + "event_time", + DataType::Timestamp( + TimeUnit::Microsecond, + Some(Arc::from("America/Los_Angeles")), + ), + true, + )])); + + let Err(error) = + align_delta_scan_output(Arc::new(EmptyExec::new(input_schema)), target_schema) + else { + return Err(DataFusionError::Plan( + "expected unrestored timestamp timezone to be rejected".to_string(), + )); + }; + + assert!( + error + .to_string() + .contains("did not restore timestamp field") + ); + Ok(()) + } +} diff --git a/crates/sail-delta-lake/src/physical/table_scan_planner.rs b/crates/sail-delta-lake/src/physical/table_scan_planner.rs index 5b08f2534..8391c664c 100644 --- a/crates/sail-delta-lake/src/physical/table_scan_planner.rs +++ b/crates/sail-delta-lake/src/physical/table_scan_planner.rs @@ -12,7 +12,7 @@ use sail_physical_plan::merge_cardinality_check::MergeCardinalityCheckExec; use crate::logical::table_source::DeltaTableSource; use crate::physical::scan_planner::plan_delta_scan; use crate::physical_plan::planner::create_row_level_write_physical_plan; -use crate::table_format::{plan_delta_write, DeltaWriteNode}; +use crate::table_format::{DeltaWriteNode, plan_delta_write}; /// Physical planner for logical Delta table scans. /// Plans `DeltaTableSource` table scans directly without an intermediate extension node. diff --git a/crates/sail-delta-lake/src/physical_plan/action_schema.rs b/crates/sail-delta-lake/src/physical_plan/action_schema.rs index eb91c9bb9..100be89a4 100644 --- a/crates/sail-delta-lake/src/physical_plan/action_schema.rs +++ b/crates/sail-delta-lake/src/physical_plan/action_schema.rs @@ -6,8 +6,8 @@ use datafusion_common::{DataFusionError, Result}; use serde::{Deserialize, Serialize}; use crate::spec::{ - add_struct_type, metadata_struct_type, protocol_struct_type, remove_struct_type, Action, Add, - DeltaOperation, Metadata, Protocol, Remove, + Action, Add, DeltaOperation, Metadata, Protocol, Remove, add_struct_type, metadata_struct_type, + protocol_struct_type, remove_struct_type, }; use crate::transaction::OperationMetrics; diff --git a/crates/sail-delta-lake/src/physical_plan/catalog_location.rs b/crates/sail-delta-lake/src/physical_plan/catalog_location.rs deleted file mode 100644 index 64d5df7e5..000000000 --- a/crates/sail-delta-lake/src/physical_plan/catalog_location.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::sync::Arc; - -use datafusion::execution::context::TaskContext; -use datafusion_common::{DataFusionError, Result}; -use sail_catalog::manager::CatalogManager; -use sail_common_datafusion::catalog::TableKind; -use sail_common_datafusion::extension::SessionExtensionAccessor; -use url::Url; - -use crate::table_format::parse_location_to_url; - -pub(crate) async fn resolve_catalog_table_url( - context: &Arc, - catalog_table: Option<&[String]>, - fallback_url: &Url, -) -> Result { - let Some(catalog_table) = catalog_table else { - return Ok(fallback_url.clone()); - }; - let manager = context.extension::()?; - let status = manager - .get_table(catalog_table) - .await - .map_err(|e| DataFusionError::External(Box::new(e)))?; - let TableKind::Table { location, .. } = status.kind else { - return Ok(fallback_url.clone()); - }; - location - .as_deref() - .map(parse_location_to_url) - .transpose() - .map(|url| url.unwrap_or_else(|| fallback_url.clone())) -} diff --git a/crates/sail-delta-lake/src/physical_plan/commit_exec.rs b/crates/sail-delta-lake/src/physical_plan/commit_exec.rs index 09d83aa67..f66d83a6f 100644 --- a/crates/sail-delta-lake/src/physical_plan/commit_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/commit_exec.rs @@ -27,32 +27,31 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties}; use futures::stream::{self, StreamExt}; use log::warn; use object_store::{Error as ObjectStoreError, ObjectStoreExt, PutMode, PutOptions}; use sail_catalog::manager::CatalogManager; -use sail_common_datafusion::catalog::LakehouseExecutionContext; +use sail_common_datafusion::catalog::{CommitAuthority, LakehouseExecutionContext}; use sail_common_datafusion::datasource::PhysicalSinkMode; use sail_common_datafusion::extension::SessionExtensionAccessor; use url::Url; use crate::catalog::coordinator::{DeltaCatalogCommitCoordinator, DeltaCatalogManagedTable}; use crate::catalog_managed::{catalog_managed_delta_table, enable_catalog_managed_create_actions}; -use crate::delta_log::{get_object_store_from_context, LogStoreRef, StorageConfig}; +use crate::delta_log::{LogStoreRef, StorageConfig, get_object_store_from_context}; use crate::physical_plan::action_schema::ExecCommitMeta; -use crate::physical_plan::catalog_location::resolve_catalog_table_url; -use crate::physical_plan::{decode_actions_and_meta_from_batch, DeltaCommitContext, COL_ACTION}; +use crate::physical_plan::{COL_ACTION, DeltaCommitContext, decode_actions_and_meta_from_batch}; use crate::schema::{ metadata_for_create_with_struct_type, normalize_delta_schema, protocol_for_create, schema_has_column_defaults, schema_has_generated_columns, schema_has_identity_columns, }; use crate::snapshot::DeltaSnapshotConfig; use crate::spec::{ - commit_path, contains_timestampntz_arrow, contains_variant_arrow, ColumnMetadataKey, - CommitAction, DeltaError, DeltaOperation, Metadata, MetadataValue, SaveMode, StatValue, Stats, - StructField, StructType, TableFeature, + ColumnMetadataKey, CommitAction, DeltaError, DeltaOperation, Metadata, MetadataValue, SaveMode, + StatValue, Stats, StructField, StructType, TableFeature, commit_path, + contains_timestampntz_arrow, contains_variant_arrow, }; use crate::table::{ create_delta_table_with_object_store, load_catalog_managed_commits_for_snapshot, @@ -378,12 +377,11 @@ impl DeltaCommitExec { if let Some(protocol) = actions.iter().find_map(|action| match action { CommitAction::Protocol(protocol) => Some(protocol), _ => None, - }) { - if protocol != snapshot.protocol() { - return Err(DataFusionError::Plan( - "Delta table already exists with a different protocol".to_string(), - )); - } + }) && protocol != snapshot.protocol() + { + return Err(DataFusionError::Plan( + "Delta table already exists with a different protocol".to_string(), + )); } if let Some(metadata) = actions.iter().find_map(|action| match action { @@ -572,8 +570,6 @@ impl ExecutionPlan for DeltaCommitExec { let future = async move { let _elapsed_compute_timer = elapsed_compute.timer(); let storage_config = StorageConfig; - let table_url = - resolve_catalog_table_url(&context, catalog_table.as_deref(), &table_url).await?; let object_store = get_object_store_from_context(&context, &table_url)?; let mut total_rows = 0u64; @@ -656,7 +652,7 @@ impl ExecutionPlan for DeltaCommitExec { log::trace!( "final_actions_len: {}, final_action_kinds: {:?}", final_actions.len(), - &kinds + kinds ); if !has_commit_payload_actions(&final_actions) { @@ -665,11 +661,13 @@ impl ExecutionPlan for DeltaCommitExec { return Ok(batch); } - let catalog_managed_table = match catalog_table.as_deref() { - Some(catalog_table) => { + let catalog_managed_table = match (catalog_table.as_deref(), lakehouse_table.as_ref()) { + (Some(catalog_table), Some(lakehouse_table)) + if lakehouse_table.commit == CommitAuthority::DeltaRatifiedCommit => + { Self::load_catalog_managed_table(&context, catalog_table, &table_url).await? } - None => None, + _ => None, }; // For new tables, always ensure Protocol + Metadata are present and use Create. // Even if the writer supplied an operation (e.g., Overwrite), the first commit @@ -873,34 +871,46 @@ impl ExecutionPlan for DeltaCommitExec { operation, operation_metrics, ) - } else if let Some(bootstrap_reference) = - Self::existing_create_bootstrap_snapshot(&log_store, &bootstrap_actions) - .await? - { - let operation = - Self::write_operation_for_sink_mode(&partition_columns, &sink_mode); - ( - Some(bootstrap_reference), - commit_actions, - operation, - operation_metrics, - ) } else { - let bootstrap_commit = CommitBuilder::from( - CommitProperties::default().with_user_metadata(user_metadata.clone()), - ) - .with_actions(bootstrap_actions) - .build(None, log_store.clone(), operation) - .await - .map_err(|e| DataFusionError::External(Box::new(e)))?; - let operation = - Self::write_operation_for_sink_mode(&partition_columns, &sink_mode); - ( - bootstrap_commit.snapshot, - commit_actions, - operation, - operation_metrics, + match Self::existing_create_bootstrap_snapshot( + &log_store, + &bootstrap_actions, ) + .await? + { + Some(bootstrap_reference) => { + let operation = Self::write_operation_for_sink_mode( + &partition_columns, + &sink_mode, + ); + ( + Some(bootstrap_reference), + commit_actions, + operation, + operation_metrics, + ) + } + _ => { + let bootstrap_commit = CommitBuilder::from( + CommitProperties::default() + .with_user_metadata(user_metadata.clone()), + ) + .with_actions(bootstrap_actions) + .build(None, log_store.clone(), operation) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let operation = Self::write_operation_for_sink_mode( + &partition_columns, + &sink_mode, + ); + ( + bootstrap_commit.snapshot, + commit_actions, + operation, + operation_metrics, + ) + } + } } } else { ( diff --git a/crates/sail-delta-lake/src/physical_plan/discovery_exec.rs b/crates/sail-delta-lake/src/physical_plan/discovery_exec.rs index b5798561d..d95c70344 100644 --- a/crates/sail-delta-lake/src/physical_plan/discovery_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/discovery_exec.rs @@ -12,7 +12,7 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr}; use futures::TryStreamExt; use url::Url; diff --git a/crates/sail-delta-lake/src/physical_plan/dv_writer_exec.rs b/crates/sail-delta-lake/src/physical_plan/dv_writer_exec.rs index a124d94d4..4e65efaa2 100644 --- a/crates/sail-delta-lake/src/physical_plan/dv_writer_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/dv_writer_exec.rs @@ -17,7 +17,7 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr}; use futures::stream::{self, StreamExt}; @@ -27,8 +27,8 @@ use url::Url; use crate::deletion_vector::{DeletionVectorBitmap, DeletionVectorWriter}; use crate::physical_plan::{ - current_timestamp_millis, decode_adds_from_batch, delta_action_schema, encode_actions, - meta_adds, ExecCommitMeta, COL_ACTION, + COL_ACTION, ExecCommitMeta, current_timestamp_millis, decode_adds_from_batch, + delta_action_schema, encode_actions, meta_adds, }; use crate::spec::{Action, Add, RemoveOptions}; use crate::transaction::OperationMetrics; @@ -559,8 +559,8 @@ impl ExecutionPlan for DeletionVectorRowsWriterExec { } } - if let Some(path) = current_path { - if let Some(stats) = write_merge_dv_actions_for_path( + if let Some(path) = current_path + && let Some(stats) = write_merge_dv_actions_for_path( path, current_bitmap, &add_by_path, @@ -571,12 +571,11 @@ impl ExecutionPlan for DeletionVectorRowsWriterExec { &mut output_actions, ) .await? - { - total_deleted_rows += stats.newly_deleted_rows; - num_dv_added += 1; - if stats.had_existing_dv { - num_dv_updated += 1; - } + { + total_deleted_rows += stats.newly_deleted_rows; + num_dv_added += 1; + if stats.had_existing_dv { + num_dv_updated += 1; } } @@ -897,7 +896,7 @@ async fn scan_file_for_matching_rows( "{}{}{}", table_root, object_store::path::DELIMITER, - &add.path + add.path )); let file_size = add.size as u64; let partitioned_file = PartitionedFile::new(file_location.to_string(), file_size); diff --git a/crates/sail-delta-lake/src/physical_plan/log_replay_exec.rs b/crates/sail-delta-lake/src/physical_plan/log_replay_exec.rs index 7180b10e3..8fbf2c138 100644 --- a/crates/sail-delta-lake/src/physical_plan/log_replay_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/log_replay_exec.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::array::{Array, ArrayRef, BooleanArray, Int64Array, StringArray}; -use datafusion::arrow::compute::{cast, concat, SortOptions}; +use datafusion::arrow::compute::{SortOptions, cast, concat}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::execution::context::TaskContext; @@ -16,9 +16,9 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties}; -use futures::{stream, TryStreamExt}; +use futures::{TryStreamExt, stream}; use url::Url; use crate::physical_plan::{COL_LOG_IS_REMOVE, COL_LOG_VERSION, COL_REPLAY_PATH}; diff --git a/crates/sail-delta-lake/src/physical_plan/metadata_stats_exec.rs b/crates/sail-delta-lake/src/physical_plan/metadata_stats_exec.rs index 71e1ccb96..69ea9b68e 100644 --- a/crates/sail-delta-lake/src/physical_plan/metadata_stats_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/metadata_stats_exec.rs @@ -3,7 +3,7 @@ use std::io::Cursor; use std::sync::Arc; use async_trait::async_trait; -use datafusion::arrow::array::{new_null_array, ArrayRef, StringArray, StructArray}; +use datafusion::arrow::array::{ArrayRef, StringArray, StructArray, new_null_array}; use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::json::ReaderBuilder as JsonReaderBuilder; @@ -15,7 +15,7 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties}; use futures::TryStreamExt; @@ -83,10 +83,10 @@ impl DeltaMetadataStatsExec { fn parse_stats_array(&self, batch: &RecordBatch) -> Result { // Priority 1: if the batch already has a typed `stats_parsed` struct column // (e.g. read from a checkpoint that persists stats in struct form), use it directly. - if let Some(existing) = batch.column_by_name(FIELD_NAME_STATS_PARSED) { - if matches!(existing.data_type(), DataType::Struct(_)) { - return Ok(Arc::clone(existing)); - } + if let Some(existing) = batch.column_by_name(FIELD_NAME_STATS_PARSED) + && matches!(existing.data_type(), DataType::Struct(_)) + { + return Ok(Arc::clone(existing)); } // Priority 2: parse from the replay pipeline's `stats_json` column. diff --git a/crates/sail-delta-lake/src/physical_plan/mod.rs b/crates/sail-delta-lake/src/physical_plan/mod.rs index 1e09ff6bb..7ebaa4072 100644 --- a/crates/sail-delta-lake/src/physical_plan/mod.rs +++ b/crates/sail-delta-lake/src/physical_plan/mod.rs @@ -24,17 +24,15 @@ use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::{ExecutionPlan, Partitioning}; -use datafusion_physical_expr::expressions::{lit, Column as PhysicalColumn}; +use datafusion_physical_expr::expressions::{Column as PhysicalColumn, lit}; mod action_schema; -mod catalog_location; mod commit_exec; pub mod discovery_exec; mod dv_writer_exec; mod log_replay_exec; mod meta_adds; mod metadata_stats_exec; -mod relaxed_tz_exec; mod remove_actions_exec; mod scan_by_adds_exec; mod write_context; @@ -42,8 +40,8 @@ mod writer_exec; mod writer_options; pub use action_schema::{ - decode_actions_and_meta_from_batch, decode_adds_from_batch, delta_action_schema, - encode_actions, ExecCommitMeta, COL_ACTION, + COL_ACTION, ExecCommitMeta, decode_actions_and_meta_from_batch, decode_adds_from_batch, + delta_action_schema, encode_actions, }; pub use commit_exec::DeltaCommitExec; pub use discovery_exec::DeltaDiscoveryExec; @@ -52,14 +50,13 @@ pub use log_replay_exec::DeltaLogReplayExec; pub use metadata_stats_exec::DeltaMetadataStatsExec; pub mod planner; pub use planner::{ - plan_delete, plan_delete_mor, plan_merge, plan_merge_mor, plan_update, DeltaPhysicalPlanner, - DeltaPlannerConfig, PlannerContext, + DeltaPhysicalPlanner, DeltaPlannerConfig, PlannerContext, plan_delete, plan_delete_mor, + plan_merge, plan_merge_mor, plan_update, }; -pub use relaxed_tz_exec::RelaxedTzCastExec; pub use remove_actions_exec::DeltaRemoveActionsExec; pub use scan_by_adds_exec::DeltaScanByAddsExec; pub use write_context::{ - prepare_delta_write_context, DeltaCommitContext, DeltaSnapshotContext, DeltaWriteContext, + DeltaCommitContext, DeltaSnapshotContext, DeltaWriteContext, prepare_delta_write_context, }; pub use writer_exec::DeltaWriterExec; pub use writer_options::DeltaWriterExecOptions; diff --git a/crates/sail-delta-lake/src/physical_plan/planner/commit.rs b/crates/sail-delta-lake/src/physical_plan/planner/commit.rs index a5401c411..981870f23 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/commit.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/commit.rs @@ -20,12 +20,12 @@ use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{DataFusionError, Result}; +use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::execution_plan::reset_plan_states; use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::union::UnionExec; -use datafusion::physical_plan::ExecutionPlan; use datafusion_common::{JoinType, NullEquality}; use datafusion_physical_expr::expressions::Column; use sail_common_datafusion::catalog::LakehouseExecutionContext; @@ -33,7 +33,7 @@ use sail_common_datafusion::datasource::PhysicalSinkMode; use url::Url; use super::context::PlannerContext; -use super::utils::{build_log_replay_pipeline_with_options, LogReplayOptions}; +use super::utils::{LogReplayOptions, build_log_replay_pipeline_with_options}; use crate::datasource::PATH_COLUMN; use crate::physical_plan::{ DeltaCommitExec, DeltaDiscoveryExec, DeltaRemoveActionsExec, DeltaWriteContext, diff --git a/crates/sail-delta-lake/src/physical_plan/planner/context.rs b/crates/sail-delta-lake/src/physical_plan/planner/context.rs index e09845ef4..b04bce6ba 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/context.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/context.rs @@ -22,15 +22,15 @@ use sail_common_datafusion::datasource::PhysicalSinkMode; use url::Url; use super::log_segment::LogSegmentFiles; -use crate::delta_log::{default_logstore, LogStoreRef, StorageConfig}; -use crate::options::gen::DeltaWriteOptions; +use crate::delta_log::{LogStoreRef, StorageConfig, default_logstore}; +use crate::options::r#gen::DeltaWriteOptions; use crate::physical_plan::{ - prepare_delta_write_context, DeltaCommitContext, DeltaWriteContext, DeltaWriterExecOptions, + DeltaCommitContext, DeltaWriteContext, DeltaWriterExecOptions, prepare_delta_write_context, }; use crate::snapshot::DeltaSnapshotConfig; use crate::table::{ - create_delta_table_with_object_store, load_catalog_managed_commits_for_snapshot, DeltaSnapshot, - DeltaTable, + DeltaSnapshot, DeltaTable, catalog_managed_commit_context, + create_delta_table_with_object_store, load_catalog_managed_commits_for_snapshot, }; /// Configuration shared by all Delta planners. @@ -318,7 +318,9 @@ impl<'a> PlannerContext<'a> { } else { let log_store = self.log_store()?; let mut table_config = table_config; - if let Some(lakehouse_table) = &self.config.lakehouse_table { + if let Some(lakehouse_table) = + catalog_managed_commit_context(self.config.lakehouse_table.as_ref()) + { table_config.catalog_managed_commits = load_catalog_managed_commits_for_snapshot( &self.session, lakehouse_table, diff --git a/crates/sail-delta-lake/src/physical_plan/planner/log_scan.rs b/crates/sail-delta-lake/src/physical_plan/planner/log_scan.rs index d923590db..a1072e815 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/log_scan.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/log_scan.rs @@ -1,19 +1,19 @@ use std::sync::{Arc, LazyLock}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use datafusion::common::{DataFusionError, Result, ScalarValue}; -use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::file_format::FileFormat; +use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::{FileGroup, FileScanConfigBuilder}; use datafusion::datasource::source::DataSourceExec; use datafusion::datasource::table_schema::TableSchema; use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::union::UnionExec; use datafusion_common::extensions::Extensions; -use futures::{stream, StreamExt, TryStreamExt}; -use object_store::path::{Path, DELIMITER}; +use futures::{StreamExt, TryStreamExt, stream}; +use object_store::path::{DELIMITER, Path}; use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; use super::context::PlannerContext; @@ -44,6 +44,59 @@ static DELTA_LOG_FILE_SCHEMA: LazyLock = LazyLock::new(|| { ])) }); +fn recursively_nullable_field(field: &Field) -> Field { + let data_type = match field.data_type() { + DataType::Struct(fields) => DataType::Struct( + fields + .iter() + .map(|field| Arc::new(recursively_nullable_field(field))) + .collect::(), + ), + DataType::List(element) => DataType::List(Arc::new(recursively_nullable_field(element))), + DataType::LargeList(element) => { + DataType::LargeList(Arc::new(recursively_nullable_field(element))) + } + DataType::FixedSizeList(element, size) => { + DataType::FixedSizeList(Arc::new(recursively_nullable_field(element)), *size) + } + DataType::Map(entries, sorted) => { + let entries_type = match entries.data_type() { + DataType::Struct(fields) if fields.len() == 2 => { + let key = Arc::clone(&fields[0]); + let value = Arc::new(recursively_nullable_field(&fields[1])); + DataType::Struct(Fields::from(vec![key, value])) + } + data_type => data_type.clone(), + }; + DataType::Map( + Arc::new( + Field::new(entries.name(), entries_type, false) + .with_metadata(entries.metadata().clone()), + ), + *sorted, + ) + } + data_type => data_type.clone(), + }; + Field::new(field.name(), data_type, true).with_metadata(field.metadata().clone()) +} + +fn replay_scan_schema(checkpoint_schema: &Schema) -> Result { + // Checkpoint schemas may omit null action fields and contain checkpoint-only parsed fields. + // JSON tails must still be decoded with every canonical action field available. + let checkpoint_schema = Schema::new_with_metadata( + checkpoint_schema + .fields() + .iter() + .map(|field| Arc::new(recursively_nullable_field(field)) as FieldRef) + .collect::>(), + checkpoint_schema.metadata().clone(), + ); + let replay_schema = + Schema::try_merge([checkpoint_schema, DELTA_LOG_FILE_SCHEMA.as_ref().clone()])?; + Ok(Arc::new(replay_schema)) +} + #[derive(Debug, Clone, Default)] pub struct LogScanOptions { /// Optional projection of top-level log columns (e.g. ["add", "remove", "metaData"]). @@ -225,13 +278,8 @@ pub async fn build_delta_log_datasource_scans_with_options( }; let has_commit_files = !commit_metas.is_empty(); - let merged = match (parquet_schema, has_commit_files) { - (Some(p), _) => { - // When a checkpoint (parquet) file is present, prefer its schema. The parquet - // checkpoint schema has proper Map types for fields like `add.partitionValues`, - // while JSON inference yields Struct types which are incompatible with `map_extract`. - p - } + let replay_schema = match (parquet_schema, has_commit_files) { + (Some(p), _) => replay_scan_schema(&p)?, (None, true) => { // No parquet checkpoint exists (e.g. before the first checkpoint interval fires). // Use the canonical Delta log file schema so that `partitionValues` and other @@ -241,7 +289,7 @@ pub async fn build_delta_log_datasource_scans_with_options( (None, false) => { return Err(DataFusionError::Plan( "no _delta_log files found to build log scan".to_string(), - )) + )); } }; @@ -250,7 +298,7 @@ pub async fn build_delta_log_datasource_scans_with_options( if !projection.iter().any(|col| col == COL_LOG_VERSION) { projection.push(COL_LOG_VERSION.to_string()); } - let file_schema_len = merged.fields().len(); + let file_schema_len = replay_schema.fields().len(); let mut indices = Vec::with_capacity(projection.len()); for col in &projection { if col == COL_LOG_VERSION { @@ -258,7 +306,7 @@ pub async fn build_delta_log_datasource_scans_with_options( continue; } - match merged.index_of(col) { + match replay_schema.index_of(col) { Ok(idx) => indices.push(idx), Err(_) => { // Some Delta writers/checkpoint formats may omit an action column @@ -275,7 +323,7 @@ pub async fn build_delta_log_datasource_scans_with_options( let target_partitions = ctx.session().config().target_partitions(); let table_schema = TableSchema::new( - Arc::clone(&merged), + Arc::clone(&replay_schema), vec![Arc::new(Field::new( COL_LOG_VERSION, DataType::Int64, @@ -368,3 +416,55 @@ pub async fn build_delta_log_datasource_union_with_options( Ok((UnionExec::try_new(inputs)?, checkpoint_files, commit_files)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[expect(clippy::expect_used)] + fn replay_schema_relaxes_checkpoint_fields_and_restores_log_fields() -> Result<()> { + let parsed = Field::new( + "partitionValues_parsed", + DataType::Struct(Fields::from(vec![Arc::new(Field::new( + "category", + DataType::Utf8, + false, + ))])), + false, + ); + let checkpoint = Schema::new(vec![Field::new( + "add", + DataType::Struct(Fields::from(vec![ + Arc::new(Field::new("path", DataType::Utf8, false)), + Arc::new(parsed), + ])), + true, + )]); + + let replay = replay_scan_schema(&checkpoint).expect("replay schema should merge"); + let add = replay + .field_with_name("add") + .expect("add should be present"); + let DataType::Struct(add_fields) = add.data_type() else { + return Err(DataFusionError::Plan( + "add should remain a struct".to_string(), + )); + }; + let parsed = add_fields + .find("partitionValues_parsed") + .map(|(_, field)| field) + .expect("parsed partition values should be present"); + let DataType::Struct(parsed_fields) = parsed.data_type() else { + return Err(DataFusionError::Plan( + "parsed partition values should remain a struct".to_string(), + )); + }; + + assert!(parsed.is_nullable()); + assert!(parsed_fields[0].is_nullable()); + assert!(add_fields.find("deletionVector").is_some()); + assert!(replay.field_with_name("remove").is_ok()); + Ok(()) + } +} diff --git a/crates/sail-delta-lake/src/physical_plan/planner/log_segment.rs b/crates/sail-delta-lake/src/physical_plan/planner/log_segment.rs index 229b78404..08d32b9f0 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/log_segment.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/log_segment.rs @@ -4,10 +4,10 @@ use datafusion::common::Result; use super::context::PlannerContext; pub use crate::delta_log::segment_files::{ - list_log_segment_files as kernel_list_log_segment_files, LogSegmentFiles, - LogSegmentResolveOptions, + LogSegmentFiles, LogSegmentResolveOptions, + list_log_segment_files as kernel_list_log_segment_files, }; -use crate::snapshot::{catalog_managed_commit_file_name, CatalogManagedCommitSet}; +use crate::snapshot::{CatalogManagedCommitSet, catalog_managed_commit_file_name}; use crate::spec::{parse_commit_version, parse_compacted_json_versions, parse_version_prefix}; /// List Delta log files up to `max_version`, using the planner-local cache when available. @@ -78,16 +78,16 @@ pub async fn resolve_log_segment_files( .iter() .filter_map(|f| parse_compacted_json_versions(f).map(|r| (r, f.clone()))) .collect(); - parsed.sort_by_key(|b| std::cmp::Reverse(b.0 .1)); // sort by end_version descending + parsed.sort_by_key(|b| std::cmp::Reverse(b.0.1)); // sort by end_version descending let mut selected: Vec<((i64, i64), String)> = Vec::new(); let mut covered_up_to: Option = None; for entry in parsed { - if let Some(boundary) = covered_up_to { - if entry.0 .1 >= boundary { - continue; - } + if let Some(boundary) = covered_up_to + && entry.0.1 >= boundary + { + continue; } - covered_up_to = Some(entry.0 .0); + covered_up_to = Some(entry.0.0); selected.push(entry); } diff --git a/crates/sail-delta-lake/src/physical_plan/planner/metadata_predicate.rs b/crates/sail-delta-lake/src/physical_plan/planner/metadata_predicate.rs index d79849753..91cdd1ba3 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/metadata_predicate.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/metadata_predicate.rs @@ -9,8 +9,8 @@ use datafusion::common::{ use datafusion::logical_expr::expr::{Between, BinaryExpr, Cast, InList}; use datafusion::logical_expr::utils::{conjunction, disjunction}; use datafusion::logical_expr::{Expr, Operator}; -use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::filter::FilterExec; use crate::datasource::simplify_expr; use crate::physical_plan::DeltaMetadataStatsExec; @@ -19,7 +19,7 @@ use crate::spec::fields::{ FIELD_NAME_STATS_PARSED, STATS_FIELD_MAX_VALUES, STATS_FIELD_MIN_VALUES, STATS_FIELD_NULL_COUNT, STATS_FIELD_NUM_RECORDS, }; -use crate::spec::{stats_schema, StructType}; +use crate::spec::{StructType, stats_schema}; use crate::table::DeltaSnapshot; pub(crate) fn predicate_requires_stats(expr: &Expr, partition_columns: &[String]) -> bool { diff --git a/crates/sail-delta-lake/src/physical_plan/planner/mod.rs b/crates/sail-delta-lake/src/physical_plan/planner/mod.rs index 5ed5e976e..c4b1d1746 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/mod.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/mod.rs @@ -33,8 +33,8 @@ mod row_level; pub use context::{DeltaPlannerConfig, PlannerContext}; pub use op_delete::{build_delete_plan as plan_delete, build_delete_plan_mor as plan_delete_mor}; pub use op_merge::{ - build_merge_plan as plan_merge, build_merge_plan_mor as plan_merge_mor, MergePredicateInfo, - OperationOverride, RowLevelTargetInfo, RowLevelWriteInfo, + MergePredicateInfo, OperationOverride, RowLevelTargetInfo, RowLevelWriteInfo, + build_merge_plan as plan_merge, build_merge_plan_mor as plan_merge_mor, }; pub use op_update::build_update_plan as plan_update; pub use row_level::create_row_level_write_physical_plan; diff --git a/crates/sail-delta-lake/src/physical_plan/planner/op_delete.rs b/crates/sail-delta-lake/src/physical_plan/planner/op_delete.rs index 2c2d4e184..f55e9c93d 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/op_delete.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/op_delete.rs @@ -24,10 +24,10 @@ use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapter use super::commit::assemble_commit_plan; use super::context::PlannerContext; use super::metadata_predicate::{build_metadata_filter, predicate_requires_stats}; -use super::utils::{build_log_replay_pipeline_with_options, LogReplayOptions}; +use super::utils::{LogReplayOptions, build_log_replay_pipeline_with_options}; use crate::physical_plan::{ - prepare_delta_write_context, DeltaCommitContext, DeltaDiscoveryExec, DeltaScanByAddsExec, - DeltaWriterExecOptions, + DeltaCommitContext, DeltaDiscoveryExec, DeltaScanByAddsExec, DeltaWriterExecOptions, + prepare_delta_write_context, }; use crate::spec::DeltaOperation; @@ -122,6 +122,7 @@ pub async fn build_delete_plan( None, None, ctx.lakehouse_table().cloned(), + snapshot_state.load_config().catalog_managed_commits.clone(), )); // Adapt the predicate to the scan schema. PhysicalExpr Column indices are schema-dependent, diff --git a/crates/sail-delta-lake/src/physical_plan/planner/op_merge.rs b/crates/sail-delta-lake/src/physical_plan/planner/op_merge.rs index 704d4041b..eda7b334e 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/op_merge.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/op_merge.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use datafusion::arrow::compute::SortOptions; -use datafusion::common::{internal_err, DataFusionError, Result}; +use datafusion::common::{DataFusionError, Result, internal_err}; use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::execution_plan::reset_plan_states; @@ -24,7 +24,7 @@ use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::{ExecutionPlan, Partitioning}; -use datafusion_common::{not_impl_err, JoinType, NullEquality}; +use datafusion_common::{JoinType, NullEquality, not_impl_err}; use datafusion_physical_expr::expressions::{Column, IsNullExpr}; use sail_common_datafusion::catalog::LakehouseExecutionContext; use sail_common_datafusion::datasource::{OptionLayer, PhysicalSinkMode, RowLevelCommand}; @@ -37,7 +37,7 @@ use super::commit::{ use super::context::PlannerContext; use super::utils::LogReplayOptions; use crate::datasource::PATH_COLUMN; -use crate::physical_plan::{prepare_delta_write_context, DeltaCommitExec, DeltaWriterExec}; +use crate::physical_plan::{DeltaCommitExec, DeltaWriterExec, prepare_delta_write_context}; use crate::spec::{DeltaOperation, MergePredicate}; /// Target table information shared by Delta row-level operations. diff --git a/crates/sail-delta-lake/src/physical_plan/planner/op_write.rs b/crates/sail-delta-lake/src/physical_plan/planner/op_write.rs index 71f517115..660cdba0b 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/op_write.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/op_write.rs @@ -28,12 +28,12 @@ use sail_common_datafusion::logical_expr::ExprWithSource; use super::context::PlannerContext; use super::metadata_predicate::{build_metadata_filter, predicate_requires_stats}; use super::utils::{ - align_schemas_for_union, build_log_replay_pipeline_with_options, build_standard_write_layers, - LogReplayOptions, + LogReplayOptions, align_schemas_for_union, build_log_replay_pipeline_with_options, + build_standard_write_layers, }; use crate::physical_plan::{ - create_projection, create_repartition, create_sort, DeltaCommitExec, DeltaDiscoveryExec, - DeltaRemoveActionsExec, DeltaScanByAddsExec, DeltaWriterExec, DeltaWriterExecOptions, + DeltaCommitExec, DeltaDiscoveryExec, DeltaRemoveActionsExec, DeltaScanByAddsExec, + DeltaWriterExec, DeltaWriterExecOptions, create_projection, create_repartition, create_sort, }; use crate::spec::{DeltaOperation, SaveMode}; use crate::table::DeltaSnapshot; @@ -340,6 +340,7 @@ async fn build_old_data_plan( None, None, ctx.lakehouse_table().cloned(), + snapshot_state.load_config().catalog_managed_commits.clone(), )); let negated_condition = Arc::new(NotExpr::new(condition)); diff --git a/crates/sail-delta-lake/src/physical_plan/planner/row_level.rs b/crates/sail-delta-lake/src/physical_plan/planner/row_level.rs index d3e1231ab..657b6edab 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/row_level.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/row_level.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use datafusion::catalog::Session; -use datafusion::common::{internal_err, not_impl_err, DataFusionError, Result}; +use datafusion::common::{DataFusionError, Result, internal_err, not_impl_err}; use datafusion::datasource::listing::ListingTableUrl; use datafusion::execution::SessionState; use datafusion::physical_plan::ExecutionPlan; @@ -13,13 +13,13 @@ use sail_data_source::resolve_listing_urls; use sail_logical_plan::merge::RowLevelWriteNode; use url::Url; -use crate::options::gen::DeltaWriteOptions; +use crate::options::r#gen::DeltaWriteOptions; use crate::physical_plan::planner::{ - plan_delete, plan_delete_mor, plan_merge, plan_merge_mor, DeltaPlannerConfig, - MergePredicateInfo, OperationOverride, PlannerContext, RowLevelTargetInfo, RowLevelWriteInfo, + DeltaPlannerConfig, MergePredicateInfo, OperationOverride, PlannerContext, RowLevelTargetInfo, + RowLevelWriteInfo, plan_delete, plan_delete_mor, plan_merge, plan_merge_mor, }; use crate::table::open_table_with_object_store; -use crate::table_format::{split_delta_write_options_and_table_properties, DeltaTableFormat}; +use crate::table_format::{DeltaTableFormat, split_delta_write_options_and_table_properties}; /// Creates a Delta physical execution plan for a unified `RowLevelWriteNode`. pub async fn create_row_level_write_physical_plan( @@ -60,7 +60,7 @@ pub async fn create_row_level_write_physical_plan( } else { None }; - let physical_deletion_vector = if let Some(plan) = node.deletion_vector_plan() { + let physical_deletion_vector = if let Some(plan) = node.row_index_delete_plan() { Some(planner.create_physical_plan(plan, ctx).await?) } else { None diff --git a/crates/sail-delta-lake/src/physical_plan/planner/utils.rs b/crates/sail-delta-lake/src/physical_plan/planner/utils.rs index 2771cbc66..ac17a2126 100644 --- a/crates/sail-delta-lake/src/physical_plan/planner/utils.rs +++ b/crates/sail-delta-lake/src/physical_plan/planner/utils.rs @@ -17,8 +17,8 @@ use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::common::{ Column as LogicalColumn, DataFusionError, Result, ScalarValue, ToDFSchema, }; -use datafusion::logical_expr::expr::{Case, Cast, ScalarFunction}; use datafusion::logical_expr::Expr; +use datafusion::logical_expr::expr::{Case, Cast, ScalarFunction}; use datafusion::physical_expr::expressions::Column; use datafusion::physical_expr::{LexOrdering, LexRequirement, PhysicalExpr, PhysicalSortExpr}; use datafusion::physical_expr_adapter::PhysicalExprAdapterFactory; @@ -36,15 +36,15 @@ use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapter use url::Url; use super::context::PlannerContext; -use super::log_scan::{build_delta_log_datasource_scans_with_options, LogScanOptions}; -use super::log_segment::{resolve_log_segment_files, LogSegmentResolveOptions}; +use super::log_scan::{LogScanOptions, build_delta_log_datasource_scans_with_options}; +use super::log_segment::{LogSegmentResolveOptions, resolve_log_segment_files}; use crate::datasource::{ - simplify_expr, COMMIT_TIMESTAMP_COLUMN, COMMIT_VERSION_COLUMN, PATH_COLUMN, + COMMIT_TIMESTAMP_COLUMN, COMMIT_VERSION_COLUMN, PATH_COLUMN, simplify_expr, }; use crate::options::DeltaLogReplayStrategy; use crate::physical_plan::{ - create_projection, create_repartition, create_sort, DeltaCommitExec, DeltaLogReplayExec, - DeltaWriterExec, DeltaWriterExecOptions, COL_LOG_IS_REMOVE, COL_LOG_VERSION, COL_REPLAY_PATH, + COL_LOG_IS_REMOVE, COL_LOG_VERSION, COL_REPLAY_PATH, DeltaCommitExec, DeltaLogReplayExec, + DeltaWriterExec, DeltaWriterExecOptions, create_projection, create_repartition, create_sort, }; use crate::spec::fields::{ FIELD_NAME_MODIFICATION_TIME, FIELD_NAME_PATH, FIELD_NAME_SIZE, FIELD_NAME_STATS, @@ -391,7 +391,7 @@ async fn build_log_replay_pipeline_with_files( other => { return Err(DataFusionError::Plan(format!( "log replay expects 'add' to be Struct, got {other}" - ))) + ))); } }; let has_add_field = |name: &str| add_struct_fields.iter().any(|f| f.name() == name); diff --git a/crates/sail-delta-lake/src/physical_plan/relaxed_tz_exec.rs b/crates/sail-delta-lake/src/physical_plan/relaxed_tz_exec.rs deleted file mode 100644 index becc79be4..000000000 --- a/crates/sail-delta-lake/src/physical_plan/relaxed_tz_exec.rs +++ /dev/null @@ -1,373 +0,0 @@ -use std::fmt; -use std::sync::Arc; - -use datafusion::arrow::datatypes::{DataType, SchemaRef}; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{Distribution, EquivalenceProperties}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, -}; -use datafusion_common::{internal_datafusion_err, Result, Statistics}; -use futures::StreamExt; -use sail_common_datafusion::array::record_batch::cast_record_batch_relaxed_tz; -use sail_common_datafusion::utils::items::ItemTaker; - -use crate::datasource::scan::map_statistics_to_schema; - -#[derive(Debug, Clone)] -pub struct RelaxedTzCastExec { - input: Arc, - schema: SchemaRef, - properties: Arc, -} - -impl RelaxedTzCastExec { - pub fn new(input: Arc, schema: SchemaRef) -> Self { - let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - input.output_partitioning().clone(), - input.pipeline_behavior(), - input.boundedness(), - )); - Self { - input, - schema, - properties, - } - } - - pub fn input(&self) -> &Arc { - &self.input - } - - fn aligned_timestamp_columns(&self) -> Vec { - let input_schema = self.input.schema(); - self.schema - .fields() - .iter() - .filter_map(|field| { - let input_field = input_schema.field_with_name(field.name()).ok()?; - match (input_field.data_type(), field.data_type()) { - (DataType::Timestamp(_, _), DataType::Timestamp(_, _)) - if input_field.data_type() != field.data_type() => - { - Some(field.name().clone()) - } - _ => None, - } - }) - .collect() - } -} - -impl DisplayAs for RelaxedTzCastExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { - let schema_changed = self.input.schema() != self.schema; - let timestamp_columns = self.aligned_timestamp_columns().join(", "); - match t { - DisplayFormatType::Default | DisplayFormatType::Verbose => { - write!( - f, - "RelaxedTzCastExec(schema_changed={}, timestamp_columns=[{}])", - schema_changed, timestamp_columns - ) - } - DisplayFormatType::TreeRender => { - write!( - f, - "schema_changed={}, timestamp_columns=[{}]", - schema_changed, timestamp_columns - ) - } - } - } -} - -impl ExecutionPlan for RelaxedTzCastExec { - fn name(&self) -> &str { - Self::static_name() - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn required_input_distribution(&self) -> Vec { - vec![Distribution::UnspecifiedDistribution] - } - - fn benefits_from_input_partitioning(&self) -> Vec { - vec![false] - } - - fn maintains_input_order(&self) -> Vec { - vec![true] - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - let input = children.one().map_err(|_| { - internal_datafusion_err!("RelaxedTzCastExec must have exactly one child") - })?; - Ok(Arc::new(Self::new(input, Arc::clone(&self.schema)))) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> Result { - let schema = Arc::clone(&self.schema); - let stream = self.input.execute(partition, context)?; - let stream = stream.map(move |batch| { - let schema = Arc::clone(&schema); - batch.and_then(|batch| cast_record_batch_relaxed_tz(&batch, &schema)) - }); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.schema), - stream, - ))) - } - - fn partition_statistics(&self, partition: Option) -> Result> { - let statistics = self.input.partition_statistics(partition)?; - if self.input.schema() == self.schema { - Ok(statistics) - } else { - Ok(Arc::new(map_statistics_to_schema( - &statistics, - &self.input.schema(), - &self.schema, - ))) - } - } -} - -#[cfg(test)] -#[expect(clippy::unwrap_used)] -mod tests { - use datafusion::arrow::array::{RecordBatch, TimestampMicrosecondArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::Partitioning; - use datafusion_common::stats::{ColumnStatistics, Precision}; - use datafusion_common::ScalarValue; - use futures::{stream, StreamExt}; - - use super::*; - - #[test] - fn relaxed_tz_cast_exec_retags_timestamp_batches() -> Result<()> { - let input_schema = Arc::new(Schema::new(vec![Field::new( - "event_time", - DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))), - true, - )])); - let target_schema = Arc::new(Schema::new(vec![Field::new( - "event_time", - DataType::Timestamp( - TimeUnit::Microsecond, - Some(Arc::from("America/Los_Angeles")), - ), - true, - )])); - let input_batch = RecordBatch::try_new( - Arc::clone(&input_schema), - vec![Arc::new( - TimestampMicrosecondArray::from(vec![Some(1_714_566_400_000_000)]) - .with_timezone_opt(Some(Arc::from("UTC"))), - )], - )?; - let input = Arc::new(TestExec::new(Arc::clone(&input_schema), vec![input_batch])); - let exec = RelaxedTzCastExec::new(input, Arc::clone(&target_schema)); - - let task_ctx = Arc::new(TaskContext::default()); - let stream = exec.execute(0, task_ctx)?; - let batches = stream.collect::>(); - let batches = futures::executor::block_on(batches); - let batch = batches.into_iter().next().unwrap()?; - - assert_eq!(batch.schema(), target_schema); - assert_eq!( - batch.column(0).data_type(), - target_schema.field(0).data_type() - ); - Ok(()) - } - - #[test] - fn relaxed_tz_cast_exec_remaps_statistics_to_output_schema() -> Result<()> { - let input_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, true), - Field::new( - "event_time", - DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))), - true, - ), - ])); - let target_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, true), - Field::new( - "event_time", - DataType::Timestamp( - TimeUnit::Microsecond, - Some(Arc::from("America/Los_Angeles")), - ), - true, - ), - ])); - let statistics = Statistics { - num_rows: Precision::Exact(42), - total_byte_size: Precision::Exact(1024), - column_statistics: vec![ - ColumnStatistics { - null_count: Precision::Exact(1), - max_value: Precision::Exact(ScalarValue::Int64(Some(9))), - min_value: Precision::Exact(ScalarValue::Int64(Some(1))), - sum_value: Precision::Absent, - distinct_count: Precision::Exact(7), - byte_size: Precision::Absent, - }, - ColumnStatistics { - null_count: Precision::Exact(0), - max_value: Precision::Exact(ScalarValue::TimestampMicrosecond( - Some(2_000_000), - Some(Arc::from("UTC")), - )), - min_value: Precision::Exact(ScalarValue::TimestampMicrosecond( - Some(1_000_000), - Some(Arc::from("UTC")), - )), - sum_value: Precision::Absent, - distinct_count: Precision::Absent, - byte_size: Precision::Absent, - }, - ], - }; - let input = Arc::new(TestExec::with_statistics( - Arc::clone(&input_schema), - vec![], - statistics, - )); - let exec = RelaxedTzCastExec::new(input, Arc::clone(&target_schema)); - - let stats = exec.partition_statistics(None)?; - - assert_eq!(stats.num_rows, Precision::Exact(42)); - assert_eq!(stats.total_byte_size, Precision::Exact(1024)); - assert_eq!(stats.column_statistics.len(), 2); - assert_eq!(stats.column_statistics[0].null_count, Precision::Exact(1)); - assert_eq!( - stats.column_statistics[0].min_value, - Precision::Exact(ScalarValue::Int64(Some(1))) - ); - assert_eq!( - stats.column_statistics[0].max_value, - Precision::Exact(ScalarValue::Int64(Some(9))) - ); - - Ok(()) - } - - #[derive(Debug)] - struct TestExec { - schema: SchemaRef, - batches: Vec, - statistics: Statistics, - properties: Arc, - } - - impl TestExec { - fn new(schema: SchemaRef, batches: Vec) -> Self { - Self::with_statistics( - schema.clone(), - batches, - Statistics::new_unknown(schema.as_ref()), - ) - } - - fn with_statistics( - schema: SchemaRef, - batches: Vec, - statistics: Statistics, - ) -> Self { - let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )); - Self { - schema, - batches, - statistics, - properties, - } - } - } - - impl DisplayAs for TestExec { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!(f, "TestExec") - } - } - - impl ExecutionPlan for TestExec { - fn name(&self) -> &str { - "TestExec" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - if !children.is_empty() { - return Err(internal_datafusion_err!( - "TestExec does not accept children" - )); - } - Ok(self) - } - - fn execute( - &self, - partition: usize, - _context: Arc, - ) -> Result { - if partition != 0 { - return Err(internal_datafusion_err!("TestExec has only one partition")); - } - let schema = Arc::clone(&self.schema); - let stream = stream::iter(self.batches.clone().into_iter().map(Ok)); - Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) - } - - fn partition_statistics(&self, partition: Option) -> Result> { - if partition.is_none() { - Ok(Arc::new(self.statistics.clone())) - } else { - Ok(Arc::new(Statistics::new_unknown(self.schema.as_ref()))) - } - } - } -} diff --git a/crates/sail-delta-lake/src/physical_plan/remove_actions_exec.rs b/crates/sail-delta-lake/src/physical_plan/remove_actions_exec.rs index 4fd09e019..0d3f39161 100644 --- a/crates/sail-delta-lake/src/physical_plan/remove_actions_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/remove_actions_exec.rs @@ -23,13 +23,13 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, Result}; +use datafusion_common::{Result, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties}; use futures::stream::{self, StreamExt}; use crate::physical_plan::{ - current_timestamp_millis, decode_adds_from_batch, delta_action_schema, encode_actions, - meta_adds, ExecCommitMeta, COL_ACTION, + COL_ACTION, ExecCommitMeta, current_timestamp_millis, decode_adds_from_batch, + delta_action_schema, encode_actions, meta_adds, }; use crate::spec::{Action, Add, Remove, RemoveOptions, Stats}; use crate::transaction::OperationMetrics; @@ -230,14 +230,10 @@ fn accumulate_touched_rows(accum: &mut Option, stats_json: Option<&str>) { return; }; match Stats::from_json_str(json) { - Ok(stats) => { - if let Ok(n) = u64::try_from(stats.num_records) { - *accum = Some(current.saturating_add(n)); - } else { - *accum = None; - } + Ok(stats) if let Ok(n) = u64::try_from(stats.num_records) => { + *accum = Some(current.saturating_add(n)); } - Err(_) => { + Ok(_) | Err(_) => { *accum = None; } } diff --git a/crates/sail-delta-lake/src/physical_plan/scan_by_adds_exec.rs b/crates/sail-delta-lake/src/physical_plan/scan_by_adds_exec.rs index 84ff2c079..bea778c8c 100644 --- a/crates/sail-delta-lake/src/physical_plan/scan_by_adds_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/scan_by_adds_exec.rs @@ -18,33 +18,33 @@ use datafusion::arrow::array::{ArrayRef, BooleanArray, Int64Array}; use datafusion::arrow::buffer::BooleanBuffer; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::execution::context::TaskContext; use datafusion::execution::SessionStateBuilder; +use datafusion::execution::context::TaskContext; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result, Statistics}; +use datafusion_common::{DataFusionError, Result, Statistics, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr}; use futures::stream::{self, StreamExt, TryStreamExt}; -use sail_common_datafusion::array::record_batch::cast_record_batch_relaxed_tz; use sail_common_datafusion::catalog::LakehouseExecutionContext; use sail_common_datafusion::extension::SessionExtensionAccessor; use sail_common_datafusion::rename::physical_plan::rename_physical_plan; use url::Url; use crate::datasource::scan::{ - file_scan_projection_for_schema, map_statistics_to_schema, sanitize_statistics_for_schema, - FileScanParams, TableStatsMode, + FileScanParams, TableStatsMode, file_scan_projection_for_schema, map_statistics_to_schema, + sanitize_statistics_for_schema, }; -use crate::datasource::{build_file_scan_config, DeltaScanConfig}; +use crate::datasource::{DeltaScanConfig, build_file_scan_config}; use crate::deletion_vector::DeletionVectorBitmap; use crate::delta_log::LogStoreRef; -use crate::physical_plan::{decode_adds_from_batch, meta_adds, COL_ACTION}; -use crate::schema::{arrow_field_physical_name, get_physical_schema}; -use crate::session_extension::{load_table_uncached, DeltaTableCache}; +use crate::physical_plan::{COL_ACTION, decode_adds_from_batch, meta_adds}; +use crate::schema::{arrow_field_physical_name, get_physical_schema, restore_logical_record_batch}; +use crate::session_extension::{DeltaTableCache, load_table_uncached, load_table_with_config}; +use crate::snapshot::{CatalogManagedCommitSet, DeltaSnapshotConfig}; use crate::spec::StructType; use crate::table::DeltaSnapshot; @@ -60,6 +60,7 @@ struct ScanByAddsStreamState { output_schema: SchemaRef, scan_config: DeltaScanConfig, lakehouse_table: Option, + catalog_managed_commits: Option, limit: Option, pushdown_filter: Option>, @@ -89,6 +90,7 @@ impl ScanByAddsStreamState { output_schema: SchemaRef, scan_config: DeltaScanConfig, lakehouse_table: Option, + catalog_managed_commits: Option, limit: Option, pushdown_filter: Option>, ) -> Self { @@ -100,6 +102,7 @@ impl ScanByAddsStreamState { output_schema, scan_config, lakehouse_table, + catalog_managed_commits, limit, pushdown_filter, table_opened: false, @@ -120,28 +123,42 @@ impl ScanByAddsStreamState { if self.table_opened { return Ok(()); } - // Prefer a session-scoped cache. This avoids leaking state across sessions / RuntimeEnvs. - // If the cache extension is not installed, fall back to no caching. - let lakehouse_table = self.lakehouse_table.as_ref(); - let cached = match self.context.as_ref().extension::() { - Ok(cache) => { - cache - .get( + let cached = if let Some(catalog_managed_commits) = self.catalog_managed_commits.clone() { + load_table_with_config( + self.context.as_ref(), + &self.table_url, + self.table_version, + DeltaSnapshotConfig { + require_files: false, + catalog_managed_commits: Some(catalog_managed_commits), + ..Default::default() + }, + ) + .await? + } else { + // Prefer a session-scoped cache. This avoids leaking state across sessions / RuntimeEnvs. + // If the cache extension is not installed, fall back to no caching. + let lakehouse_table = self.lakehouse_table.as_ref(); + match self.context.as_ref().extension::() { + Ok(cache) => { + cache + .get( + self.context.as_ref(), + &self.table_url, + self.table_version, + lakehouse_table, + ) + .await? + } + Err(_) => { + load_table_uncached( self.context.as_ref(), &self.table_url, self.table_version, lakehouse_table, ) .await? - } - Err(_) => { - load_table_uncached( - self.context.as_ref(), - &self.table_url, - self.table_version, - lakehouse_table, - ) - .await? + } } }; @@ -165,7 +182,7 @@ impl ScanByAddsStreamState { let kschema_arc = snapshot_state.schema(); let logical_kernel = StructType::try_from(kschema_arc) .map_err(|e| DataFusionError::External(Box::new(e)))?; - let physical_arrow = get_physical_schema(&logical_kernel, kmode); + let physical_arrow = get_physical_schema(&logical_kernel, kmode)?; let physical_partition_cols: std::collections::HashSet = table_partition_cols .iter() .map(|col| { @@ -221,6 +238,7 @@ impl ScanByAddsStreamState { .snapshot .as_deref() .ok_or_else(|| DataFusionError::Internal("missing snapshot".into()))?; + let column_mapping_mode = snapshot.effective_column_mapping_mode(); let log_store = self .log_store .as_ref() @@ -305,8 +323,11 @@ impl ScanByAddsStreamState { .and_then(move |batch| { let output_schema = Arc::clone(&output_schema); async move { - let casted = cast_record_batch_relaxed_tz(&batch, &output_schema) - .map_err(|e| DataFusionError::External(Box::new(e)))?; + let casted = restore_logical_record_batch( + &batch, + &output_schema, + column_mapping_mode, + )?; Ok(casted) } }); @@ -385,8 +406,8 @@ impl ScanByAddsStreamState { .and_then(move |batch| { let output_schema = Arc::clone(&output_schema); async move { - let casted = cast_record_batch_relaxed_tz(&batch, &output_schema) - .map_err(|e| DataFusionError::External(Box::new(e)))?; + let casted = + restore_logical_record_batch(&batch, &output_schema, column_mapping_mode)?; Ok(casted) } }); @@ -594,6 +615,7 @@ pub struct DeltaScanByAddsExec { limit: Option, pushdown_filter: Option>, lakehouse_table: Option, + catalog_managed_commits: Option, statistics: Statistics, cache: Arc, } @@ -611,6 +633,7 @@ impl DeltaScanByAddsExec { limit: Option, pushdown_filter: Option>, lakehouse_table: Option, + catalog_managed_commits: Option, ) -> Self { let statistics = Statistics::new_unknown(output_schema.as_ref()); let cache = Self::compute_properties( @@ -628,6 +651,7 @@ impl DeltaScanByAddsExec { limit, pushdown_filter, lakehouse_table, + catalog_managed_commits, statistics, cache, } @@ -701,6 +725,10 @@ impl DeltaScanByAddsExec { self.lakehouse_table.as_ref() } + pub fn catalog_managed_commits(&self) -> Option<&CatalogManagedCommitSet> { + self.catalog_managed_commits.as_ref() + } + pub fn statistics(&self) -> &Statistics { &self.statistics } @@ -760,6 +788,7 @@ impl ExecutionPlan for DeltaScanByAddsExec { let output_schema = self.schema(); let scan_config = self.scan_config.clone(); let lakehouse_table = self.lakehouse_table.clone(); + let catalog_managed_commits = self.catalog_managed_commits.clone(); let limit = self.limit; let pushdown_filter = self.pushdown_filter.clone(); let state = ScanByAddsStreamState::new( @@ -770,6 +799,7 @@ impl ExecutionPlan for DeltaScanByAddsExec { Arc::clone(&output_schema), scan_config, lakehouse_table, + catalog_managed_commits, limit, pushdown_filter, ); @@ -901,8 +931,8 @@ mod tests { use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::ExecutionPlan; + use datafusion::physical_plan::empty::EmptyExec; use datafusion_common::stats::{ColumnStatistics, Precision, Statistics}; use datafusion_common::{DataFusionError, Result, ScalarValue}; use url::Url; @@ -1017,6 +1047,7 @@ mod tests { None, None, None, + None, ) .with_table_statistics(Some(table_stats)); @@ -1071,6 +1102,7 @@ mod tests { None, None, None, + None, ) .with_output_statistics(Some(output_stats)); diff --git a/crates/sail-delta-lake/src/physical_plan/write_context.rs b/crates/sail-delta-lake/src/physical_plan/write_context.rs index 07b207087..e2c26da49 100644 --- a/crates/sail-delta-lake/src/physical_plan/write_context.rs +++ b/crates/sail-delta-lake/src/physical_plan/write_context.rs @@ -5,7 +5,7 @@ use datafusion::arrow::datatypes::{Field, Schema, SchemaRef}; use datafusion_common::{DataFusionError, Result}; use sail_common_datafusion::column_features::SAIL_WRITE_TARGET_NULLABLE_METADATA_KEY; use sail_common_datafusion::datasource::{ - PhysicalSinkMode, MERGE_SOURCE_METRIC_COLUMN, OPERATION_COLUMN, + MERGE_SOURCE_METRIC_COLUMN, OPERATION_COLUMN, PhysicalSinkMode, }; use serde::{Deserialize, Serialize}; use url::Url; @@ -20,12 +20,12 @@ use crate::schema::{ is_supported_type_change_for_schema_evolution, metadata_for_create_with_struct_type, normalize_delta_schema, protocol_can_write_type_widening, protocol_for_create, schema_contains_type_widening_metadata, schema_has_column_defaults, - schema_has_generated_columns, schema_has_identity_columns, + schema_has_generated_columns, schema_has_identity_columns, strip_column_mapping_metadata, }; use crate::snapshot::DeltaSnapshotConfig; use crate::spec::{ - contains_timestampntz_arrow, contains_variant_arrow, Action, ColumnMappingMode, DeltaOperation, - DomainMetadata, Metadata, Protocol, SaveMode, StructType, TableProperties, Transaction, + Action, ColumnMappingMode, DeltaOperation, DomainMetadata, Metadata, Protocol, SaveMode, + StructType, TableProperties, Transaction, contains_timestampntz_arrow, contains_variant_arrow, }; use crate::table::DeltaSnapshot; @@ -121,7 +121,7 @@ impl DeltaWriteContext { Ok(Arc::new(get_physical_schema( logical_kernel, self.effective_column_mapping_mode, - ))) + )?)) } } } @@ -145,10 +145,11 @@ pub fn prepare_delta_write_context( input_schema: &SchemaRef, operation_override: Option, ) -> Result { - let input_schema = normalize_delta_schema(&apply_target_nullability( - &schema_without_writer_metric_columns(input_schema), - &options.target_nullability, - )); + let input_schema = + strip_column_mapping_metadata(&normalize_delta_schema(&apply_target_nullability( + &schema_without_writer_metric_columns(input_schema), + &options.target_nullability, + ))); let mut initial_actions: Vec = Vec::new(); let planned_operation = operation_for_sink_mode(table_url, partition_columns, sink_mode); @@ -577,3 +578,61 @@ fn validate_schema_compatibility(table_schema: &Schema, input_schema: &Schema) - } Ok(()) } + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::panic)] +mod tests { + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + + use super::*; + use crate::spec::{DataType, MetadataValue, StructField}; + + fn mapped_field(name: &str, physical_name: &str, id: i64, data_type: DataType) -> StructField { + StructField::new(name, data_type, true).with_metadata([ + ("delta.columnMapping.id", MetadataValue::Number(id)), + ( + "delta.columnMapping.physicalName", + MetadataValue::String(physical_name.to_string()), + ), + ]) + } + + #[test] + fn writer_schema_preserves_nested_parquet_field_ids() -> Result<()> { + let nested = StructType::try_new(vec![mapped_field("rate", "col-rate", 2, DataType::LONG)]) + .expect("nested schema"); + let logical = StructType::try_new(vec![mapped_field( + "details", + "col-details", + 1, + DataType::Struct(Box::new(nested)), + )]) + .expect("logical schema"); + let context = DeltaWriteContext { + commit_context: DeltaCommitContext::default(), + final_schema: logical.clone(), + effective_column_mapping_mode: ColumnMappingMode::Name, + initial_actions: Vec::new(), + schema_actions: Vec::new(), + operation: None, + logical_kernel_for_mapping: Some(logical), + physical_partition_columns: Vec::new(), + }; + + let writer_schema = context.writer_schema()?; + let details = writer_schema.field_with_name("col-details")?; + let datafusion::arrow::datatypes::DataType::Struct(children) = details.data_type() else { + panic!("details must be a struct"); + }; + let rate = children.first().expect("nested rate field"); + + assert_eq!(rate.name(), "col-rate"); + assert_eq!( + rate.metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .map(String::as_str), + Some("2") + ); + Ok(()) + } +} diff --git a/crates/sail-delta-lake/src/physical_plan/writer_exec.rs b/crates/sail-delta-lake/src/physical_plan/writer_exec.rs index 0f99881b0..85af9e0a3 100644 --- a/crates/sail-delta-lake/src/physical_plan/writer_exec.rs +++ b/crates/sail-delta-lake/src/physical_plan/writer_exec.rs @@ -27,7 +27,7 @@ use datafusion::arrow::array::{ Array, ArrayRef, BooleanArray, BooleanBuilder, Int32Array, Int64Array, PrimitiveArray, UInt64Array, }; -use datafusion::arrow::compute::{filter_record_batch, SortOptions}; +use datafusion::arrow::compute::{SortOptions, filter_record_batch}; use datafusion::arrow::datatypes::{ ArrowTimestampType, DataType, Schema, SchemaRef, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, @@ -43,28 +43,28 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::{Distribution, EquivalenceProperties}; -use futures::stream::{once, StreamExt}; +use futures::stream::{StreamExt, once}; +use sail_common_datafusion::array::record_batch::cast_array_recursively; use sail_common_datafusion::catalog::LakehouseExecutionContext; use sail_common_datafusion::datasource::{ - PhysicalSinkMode, RowLevelOperationType, MERGE_SOURCE_METRIC_COLUMN, OPERATION_COLUMN, + MERGE_SOURCE_METRIC_COLUMN, OPERATION_COLUMN, PhysicalSinkMode, RowLevelOperationType, }; use url::Url; use crate::conversion::DeltaTypeConverter; use crate::delta_log::get_object_store_from_context; -use crate::physical_plan::catalog_location::resolve_catalog_table_url; use crate::physical_plan::writer_options::DeltaWriterExecOptions; use crate::physical_plan::{ - delta_action_schema, encode_actions, DeltaWriteContext, ExecCommitMeta, + DeltaWriteContext, ExecCommitMeta, delta_action_schema, encode_actions, }; +use crate::schema::adapt_array_to_physical_field; use crate::spec::{ - Action, ColumnMappingMode, DeltaOperation, Metadata, Protocol, StructType, TableFeature, - TableProperties, + Action, ColumnMappingMode, DeltaOperation, Metadata, Protocol, TableFeature, TableProperties, }; use crate::transaction::OperationMetrics; -use crate::writer::variant_shredding::{variant_top_level_columns, VariantShreddingConfig}; +use crate::writer::variant_shredding::{VariantShreddingConfig, variant_top_level_columns}; use crate::writer::{DeltaWriter, WriterConfig}; /// Counts internal row intent tags before they are stripped from writer input. @@ -294,21 +294,68 @@ pub struct DeltaWriterExec { cache: Arc, } -impl DeltaWriterExec { - /// Build a map from physical field name to logical name for top-level columns - fn build_physical_to_logical_map( - logical_kernel: &StructType, - column_mapping_mode: ColumnMappingMode, - ) -> HashMap { - let mut map = HashMap::new(); - for kf in logical_kernel.fields() { - map.insert( - kf.physical_name(column_mapping_mode).to_string(), - kf.name().clone(), - ); +fn matching_nested_types(source: &DataType, target: &DataType) -> bool { + matches!( + (source, target), + (DataType::Struct(_), DataType::Struct(_)) + | (DataType::List(_), DataType::List(_)) + | (DataType::LargeList(_), DataType::LargeList(_)) + | (DataType::FixedSizeList(_, _), DataType::FixedSizeList(_, _)) + | (DataType::Map(_, _), DataType::Map(_, _)) + ) +} + +fn validate_cast_safety_recursively( + source: &DataType, + target: &DataType, + path: &str, +) -> Result<()> { + if source == target { + return Ok(()); + } + + match (source, target) { + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + for target_field in target_fields { + if let Some(source_field) = source_fields + .iter() + .find(|field| field.name() == target_field.name()) + { + validate_cast_safety_recursively( + source_field.data_type(), + target_field.data_type(), + &format!("{path}.{}", target_field.name()), + )?; + } + } + Ok(()) } - map + (DataType::List(source), DataType::List(target)) + | (DataType::LargeList(source), DataType::LargeList(target)) => { + validate_cast_safety_recursively( + source.data_type(), + target.data_type(), + &format!("{path}[]"), + ) + } + ( + DataType::FixedSizeList(source, source_len), + DataType::FixedSizeList(target, target_len), + ) if source_len == target_len => validate_cast_safety_recursively( + source.data_type(), + target.data_type(), + &format!("{path}[]"), + ), + (DataType::Map(source, _), DataType::Map(target, _)) => validate_cast_safety_recursively( + source.data_type(), + target.data_type(), + &format!("{path}{{}}"), + ), + _ => DeltaTypeConverter::validate_cast_safety(source, target, path), } +} + +impl DeltaWriterExec { #[expect(clippy::too_many_arguments)] pub fn new( input: Arc, @@ -613,7 +660,6 @@ impl DeltaWriterExec { let elapsed_compute = MetricBuilder::new(&self.metrics).elapsed_compute(partition); let table_url = self.table_url.clone(); - let catalog_table = self.catalog_table().map(<[String]>::to_vec); let options = self.options.clone(); let partition_columns = self.partition_columns.clone(); let sink_mode = self.sink_mode.clone(); @@ -636,8 +682,6 @@ impl DeltaWriterExec { } = &options; let timezone = session_timezone; - let table_url = - resolve_catalog_table_url(&context, catalog_table.as_deref(), &table_url).await?; let object_store = get_object_store_from_context(&context, &table_url)?; match &sink_mode { @@ -694,12 +738,12 @@ impl DeltaWriterExec { let writer_path = object_store::path::Path::from(table_url.path()); let mut writer = DeltaWriter::new(object_store.clone(), writer_path, writer_config); - // Compute physical-to-logical mapping once before the loop - let phys_to_logical = logical_kernel_for_mapping.as_ref().map(|logical_kernel| { - let map = Self::build_physical_to_logical_map(logical_kernel, kernel_mode); - log::trace!("phys_to_logical: {:?}", &map); - map - }); + let logical_schema_for_mapping = logical_kernel_for_mapping + .as_ref() + .map(Schema::try_from) + .transpose() + .map_err(|e| DataFusionError::External(Box::new(e)))? + .map(Arc::new); let mut total_rows = 0u64; let mut data = stream; @@ -732,14 +776,15 @@ impl DeltaWriterExec { .collect(); log::trace!( "input_batch_fields: {:?}, target_fields: {:?}", - &input_names, - &target_names + input_names, + target_names ); let validated_batch = Self::validate_and_adapt_batch( batch, &writer_schema, - phys_to_logical.as_ref(), + logical_schema_for_mapping.as_ref(), + kernel_mode, timezone.as_deref(), )?; @@ -967,45 +1012,70 @@ impl DeltaWriterExec { fn validate_and_adapt_batch( batch: RecordBatch, final_schema: &SchemaRef, - phys_to_logical: Option<&HashMap>, + logical_schema: Option<&SchemaRef>, + column_mapping_mode: ColumnMappingMode, timezone: Option<&str>, ) -> Result { let batch_schema = batch.schema(); + if column_mapping_mode != ColumnMappingMode::None { + let logical_schema = logical_schema.ok_or_else(|| { + DataFusionError::Internal( + "column-mapped Delta write is missing its logical schema".to_string(), + ) + })?; + if logical_schema.fields().len() != final_schema.fields().len() { + return Err(DataFusionError::Plan(format!( + "logical writer schema field count {} differs from physical field count {}", + logical_schema.fields().len(), + final_schema.fields().len() + ))); + } + } // If schemas are identical, no adaptation needed - if batch_schema.as_ref() == final_schema.as_ref() { + if batch_schema.as_ref() == final_schema.as_ref() + && batch + .columns() + .iter() + .zip(final_schema.fields()) + .all(|(column, field)| column.data_type() == field.data_type()) + { return Ok(batch); } // Check if all required fields are present and types are compatible let mut adapted_columns = Vec::with_capacity(final_schema.fields().len()); - for final_field in final_schema.fields() { - // For physical writer schema, final_field.name() is physical. Map to logical if provided - let lookup_name: &str = if let Some(map) = phys_to_logical { - map.get(final_field.name().as_str()) - .map(|s| s.as_str()) - .unwrap_or(final_field.name()) - } else { - final_field.name() - }; + for (final_index, final_field) in final_schema.fields().iter().enumerate() { + let logical_field = logical_schema.and_then(|schema| schema.fields().get(final_index)); + let lookup_name = logical_field + .map(|field| field.name().as_str()) + .unwrap_or_else(|| final_field.name()); match batch_schema.column_with_name(lookup_name) { - Some((batch_index, batch_field)) => { + Some((batch_index, _)) => { let batch_column = batch.column(batch_index); - - if batch_field.data_type() == final_field.data_type() { + let validation_field = logical_field.unwrap_or(final_field); + validate_cast_safety_recursively( + batch_column.data_type(), + validation_field.data_type(), + validation_field.name(), + )?; + + if column_mapping_mode != ColumnMappingMode::None + && let Some(logical_field) = logical_field + && matching_nested_types(logical_field.data_type(), final_field.data_type()) + { + adapted_columns.push(adapt_array_to_physical_field( + batch_column, + logical_field, + final_field, + )?); + } else if batch_column.data_type() == final_field.data_type() { adapted_columns.push(batch_column.clone()); } else { - // Types don't match, validate cast safety and attempt casting - DeltaTypeConverter::validate_cast_safety( - batch_field.data_type(), - final_field.data_type(), - final_field.name(), - )?; - let casted_column = - match (batch_field.data_type(), final_field.data_type(), timezone) { + match (batch_column.data_type(), final_field.data_type(), timezone) { ( DataType::Timestamp(unit_from, Some(_)), DataType::Timestamp(unit_to, Some(target_tz)), @@ -1057,11 +1127,7 @@ impl DeltaWriterExec { ) .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))? } - _ => datafusion::arrow::compute::cast( - batch_column, - final_field.data_type(), - ) - .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?, + _ => cast_array_recursively(batch_column, final_field.data_type())?, }; adapted_columns.push(casted_column); } @@ -1133,3 +1199,121 @@ impl DisplayAs for DeltaWriterExec { } } } + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use std::collections::HashMap; + + use datafusion::arrow::array::{Array, ArrayRef, Int32Array, StructArray}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatchOptions; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + + use super::*; + + fn field_with_id(name: &str, metadata_key: &str, id: i64) -> Arc { + Arc::new( + Field::new(name, DataType::Int32, true) + .with_metadata(HashMap::from([(metadata_key.to_string(), id.to_string())])), + ) + } + + #[test] + fn adapt_batch_replaces_nested_field_metadata() -> Result<()> { + let source_fields = vec![ + field_with_id("rate", PARQUET_FIELD_ID_META_KEY, 1), + field_with_id("exponent", PARQUET_FIELD_ID_META_KEY, 2), + ]; + let details = Arc::new(StructArray::new( + source_fields.into(), + vec![ + Arc::new(Int32Array::from(vec![Some(10)])), + Arc::new(Int32Array::from(vec![Some(2)])), + ], + None, + )) as ArrayRef; + let logical_fields = vec![ + field_with_id("rate", "delta.columnMapping.id", 2), + field_with_id("exponent", "delta.columnMapping.id", 3), + ]; + let logical_schema = Arc::new(Schema::new(vec![Field::new( + "details", + DataType::Struct(logical_fields.into()), + true, + )])); + let target_fields = vec![ + field_with_id("col-rate", PARQUET_FIELD_ID_META_KEY, 2), + field_with_id("col-exponent", PARQUET_FIELD_ID_META_KEY, 3), + ]; + let final_schema = Arc::new(Schema::new(vec![Field::new( + "col-details", + DataType::Struct(target_fields.clone().into()), + true, + )])); + let batch = RecordBatch::try_new_with_options( + logical_schema.clone(), + vec![details], + &RecordBatchOptions::new().with_match_field_names(false), + )?; + + let adapted = DeltaWriterExec::validate_and_adapt_batch( + batch, + &final_schema, + Some(&logical_schema), + ColumnMappingMode::Name, + None, + )?; + let details = adapted + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(adapted.schema(), final_schema); + RecordBatch::try_new(adapted.schema(), adapted.columns().to_vec())?; + assert!(details.fields().iter().eq(target_fields.iter())); + assert_eq!( + details + .column_by_name("col-rate") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 10, + ); + assert_eq!( + details + .column_by_name("col-exponent") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 2, + ); + Ok(()) + } + + #[test] + fn nested_cast_validation_rejects_unsafe_leaf_cast() { + let source = DataType::Struct( + vec![Field::new("value", DataType::Decimal128(20, 4), true)] + .into_iter() + .map(Arc::new) + .collect(), + ); + let target = DataType::Struct( + vec![Field::new("value", DataType::Decimal128(10, 2), true)] + .into_iter() + .map(Arc::new) + .collect(), + ); + + let error = validate_cast_safety_recursively(&source, &target, "details") + .expect_err("narrowing nested cast must fail"); + + assert!(error.to_string().contains("details.value")); + } +} diff --git a/crates/sail-delta-lake/src/physical_plan/writer_options.rs b/crates/sail-delta-lake/src/physical_plan/writer_options.rs index 928eeae27..2c59a5ece 100644 --- a/crates/sail-delta-lake/src/physical_plan/writer_options.rs +++ b/crates/sail-delta-lake/src/physical_plan/writer_options.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use sail_common_datafusion::catalog::CatalogTableColumnIdentity; use serde::{Deserialize, Serialize}; -use crate::options::gen::DeltaWriteOptions; +use crate::options::r#gen::DeltaWriteOptions; /// Options for the Delta Lake writer execution plan. /// diff --git a/crates/sail-delta-lake/src/schema/arrow_conversions.rs b/crates/sail-delta-lake/src/schema/arrow_conversions.rs index 46740af52..170735e0a 100644 --- a/crates/sail-delta-lake/src/schema/arrow_conversions.rs +++ b/crates/sail-delta-lake/src/schema/arrow_conversions.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use arrow_schema::extension::{ - ExtensionType, EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, + EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType, }; use datafusion::arrow::datatypes::{ DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema, diff --git a/crates/sail-delta-lake/src/schema/converter.rs b/crates/sail-delta-lake/src/schema/converter.rs index 5c41d9c6d..770e87450 100644 --- a/crates/sail-delta-lake/src/schema/converter.rs +++ b/crates/sail-delta-lake/src/schema/converter.rs @@ -1,9 +1,20 @@ -use std::collections::HashMap; use std::sync::Arc; +use datafusion::arrow::array::{ + Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, MapArray, StructArray, + new_null_array, +}; +use datafusion::arrow::buffer::NullBuffer; use datafusion::arrow::datatypes::{ DataType as ArrowDataType, Field, Fields, Schema as ArrowSchema, SchemaRef, }; +use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datafusion_common::{DataFusionError, Result, exec_err}; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use sail_common_datafusion::array::record_batch::cast_array_recursively; +use sail_common_datafusion::schema_evolution::{ + StructFieldMatching, cast_array_with_schema_evolution_relaxed_tz, +}; use crate::spec::{ ColumnMappingMode, ColumnMetadataKey, DataType, DeltaError as DeltaTableError, DeltaResult, @@ -36,19 +47,26 @@ pub fn arrow_schema_from_struct_type( Ok(Arc::new(ArrowSchema::new(fields))) } -pub fn get_physical_arrow_schema(logical: &StructType, mode: ColumnMappingMode) -> ArrowSchema { +/// Build the physical Arrow schema used for Delta Parquet data files. +/// +/// Name mode strips Delta-side field ID metadata, then re-adds each ID under Arrow's +/// `PARQUET:field_id` key so nested IDs reach the Parquet footer. +pub fn get_physical_arrow_schema( + logical: &StructType, + mode: ColumnMappingMode, +) -> DeltaResult { + // Build the stripped Delta physical schema first, then restore IDs only for Parquet. let physical_kernel = logical.make_physical(mode); - let physical_arrow: ArrowSchema = - ArrowSchema::try_from(&physical_kernel).unwrap_or_else(|_| ArrowSchema::empty()); + let physical_arrow = ArrowSchema::try_from(&physical_kernel)?; match mode { ColumnMappingMode::Name | ColumnMappingMode::Id => { enrich_arrow_with_parquet_field_ids(&physical_arrow, logical) } - ColumnMappingMode::None => physical_arrow, + ColumnMappingMode::None => Ok(physical_arrow), } } -/// Apply Delta column mapping to an Arrow schema, renaming logical→physical column names. +/// Build a physical schema for metadata and statistics without Parquet footer field IDs. /// /// This is the Arrow-native equivalent of `StructType::make_physical`. It reads the /// `delta.columnMapping.physicalName` metadata from each Arrow field and renames the @@ -62,6 +80,83 @@ pub fn make_physical_arrow_schema(logical: &ArrowSchema, mode: ColumnMappingMode ArrowSchema::new(new_fields).with_metadata(logical.metadata().clone()) } +pub fn attach_column_mapping_metadata(requested: &ArrowSchema, table: &ArrowSchema) -> ArrowSchema { + let fields = requested + .fields() + .iter() + .map(|requested| { + table + .field_with_name(requested.name()) + .map(|table| attach_field_mapping_metadata(requested, table)) + .unwrap_or_else(|_| requested.as_ref().clone()) + }) + .collect::>(); + ArrowSchema::new(fields).with_metadata(requested.metadata().clone()) +} + +fn attach_field_mapping_metadata(requested: &Field, table: &Field) -> Field { + let mut metadata = requested.metadata().clone(); + for key in [ + ColumnMetadataKey::ColumnMappingPhysicalName.as_ref(), + ColumnMetadataKey::ColumnMappingId.as_ref(), + ColumnMetadataKey::ParquetFieldId.as_ref(), + PARQUET_FIELD_ID_META_KEY, + ] { + if let Some(value) = table.metadata().get(key) { + metadata.insert(key.to_string(), value.clone()); + } + } + + requested + .clone() + .with_data_type(attach_data_type_mapping_metadata( + requested.data_type(), + table.data_type(), + )) + .with_metadata(metadata) +} + +fn attach_data_type_mapping_metadata( + requested: &ArrowDataType, + table: &ArrowDataType, +) -> ArrowDataType { + match (requested, table) { + (ArrowDataType::Struct(requested), ArrowDataType::Struct(table)) => { + let fields = requested + .iter() + .map(|requested| { + table + .iter() + .find(|table| table.name() == requested.name()) + .map(|table| attach_field_mapping_metadata(requested, table)) + .unwrap_or_else(|| requested.as_ref().clone()) + }) + .collect::>(); + ArrowDataType::Struct(fields.into()) + } + (ArrowDataType::List(requested), ArrowDataType::List(table)) => { + ArrowDataType::List(Arc::new(attach_field_mapping_metadata(requested, table))) + } + (ArrowDataType::LargeList(requested), ArrowDataType::LargeList(table)) => { + ArrowDataType::LargeList(Arc::new(attach_field_mapping_metadata(requested, table))) + } + ( + ArrowDataType::FixedSizeList(requested, requested_len), + ArrowDataType::FixedSizeList(table, _), + ) => ArrowDataType::FixedSizeList( + Arc::new(attach_field_mapping_metadata(requested, table)), + *requested_len, + ), + (ArrowDataType::Map(requested, sorted), ArrowDataType::Map(table, _)) => { + ArrowDataType::Map( + Arc::new(attach_field_mapping_metadata(requested, table)), + *sorted, + ) + } + _ => requested.clone(), + } +} + /// Get the physical name of an Arrow field under a given column mapping mode. /// /// This is the Arrow-native equivalent of `StructField::physical_name`. @@ -76,6 +171,250 @@ pub fn arrow_field_physical_name(field: &Field, mode: ColumnMappingMode) -> &str } } +pub fn restore_logical_record_batch( + batch: &RecordBatch, + target_schema: &SchemaRef, + mode: ColumnMappingMode, +) -> Result { + if batch.schema_ref() == target_schema { + return Ok(batch.clone()); + } + + let matching = match mode { + ColumnMappingMode::None => StructFieldMatching::Name, + ColumnMappingMode::Name => StructFieldMatching::PhysicalName, + ColumnMappingMode::Id => StructFieldMatching::FieldId, + }; + let source_schema = batch.schema(); + let columns = target_schema + .fields() + .iter() + .map(|target| match source_schema.index_of(target.name()) { + Ok(index) => cast_array_with_schema_evolution_relaxed_tz( + batch.column(index), + target.as_ref(), + &datafusion_common::format::DEFAULT_CAST_OPTIONS, + matching, + ), + Err(_) if target.is_nullable() => { + Ok(new_null_array(target.data_type(), batch.num_rows())) + } + Err(_) => Err(DataFusionError::Plan(format!( + "missing required column '{}' while restoring Delta logical schema", + target.name() + ))), + }) + .collect::>>()?; + + if columns.is_empty() { + Ok(RecordBatch::try_new_with_options( + Arc::clone(target_schema), + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + )?) + } else { + Ok(RecordBatch::try_new(Arc::clone(target_schema), columns)?) + } +} + +pub(crate) fn adapt_array_to_physical_field( + source: &ArrayRef, + logical_field: &Field, + physical_field: &Field, +) -> Result { + adapt_array_to_physical_field_inner(source, logical_field, physical_field, None) +} + +fn adapt_array_to_physical_field_inner( + source: &ArrayRef, + logical_field: &Field, + physical_field: &Field, + parent_nulls: Option<&NullBuffer>, +) -> Result { + if !physical_field.is_nullable() + && (0..source.len()).any(|index| { + parent_nulls.is_none_or(|nulls| nulls.is_valid(index)) + && (source.data_type() == &ArrowDataType::Null || source.is_null(index)) + }) + { + return exec_err!( + "required field '{}' contains null values", + logical_field.name() + ); + } + + if source.data_type() == &ArrowDataType::Null + || (!source.is_empty() && source.null_count() == source.len()) + { + return Ok(new_null_array(physical_field.data_type(), source.len())); + } + + match (logical_field.data_type(), physical_field.data_type()) { + (ArrowDataType::Struct(logical_fields), ArrowDataType::Struct(physical_fields)) => { + let Some(source_struct) = source.as_any().downcast_ref::() else { + return exec_err!("expected struct array for field '{}'", logical_field.name()); + }; + if logical_fields.len() != physical_fields.len() { + return exec_err!( + "cannot adapt struct field '{}': logical field count {} differs from physical field count {}", + logical_field.name(), + logical_fields.len(), + physical_fields.len() + ); + } + let arrays = logical_fields + .iter() + .zip(physical_fields) + .map( + |(logical, physical)| match source_struct.column_by_name(logical.name()) { + Some(source) => adapt_array_to_physical_field_inner( + source, + logical.as_ref(), + physical.as_ref(), + source_struct.nulls(), + ), + None if physical.is_nullable() + || source_struct + .nulls() + .is_some_and(|nulls| nulls.null_count() == source_struct.len()) => + { + Ok(new_null_array(physical.data_type(), source_struct.len())) + } + None => exec_err!( + "required nested field '{}' is missing from input", + logical.name() + ), + }, + ) + .collect::>>()?; + Ok(Arc::new(StructArray::new( + physical_fields.clone(), + arrays, + source_struct.nulls().cloned(), + ))) + } + (ArrowDataType::List(logical), ArrowDataType::List(physical)) => { + let Some(source_list) = source.as_any().downcast_ref::() else { + return exec_err!("expected list array for field '{}'", logical_field.name()); + }; + let values = + adapt_array_to_physical_field_inner(source_list.values(), logical, physical, None)?; + Ok(Arc::new(ListArray::new( + Arc::clone(physical), + source_list.offsets().clone(), + values, + source_list.nulls().cloned(), + ))) + } + (ArrowDataType::LargeList(logical), ArrowDataType::LargeList(physical)) => { + let Some(source_list) = source.as_any().downcast_ref::() else { + return exec_err!( + "expected large list array for field '{}'", + logical_field.name() + ); + }; + let values = + adapt_array_to_physical_field_inner(source_list.values(), logical, physical, None)?; + Ok(Arc::new(LargeListArray::new( + Arc::clone(physical), + source_list.offsets().clone(), + values, + source_list.nulls().cloned(), + ))) + } + ( + ArrowDataType::FixedSizeList(logical, logical_len), + ArrowDataType::FixedSizeList(physical, physical_len), + ) => { + let Some(source_list) = source.as_any().downcast_ref::() else { + return exec_err!( + "expected fixed-size list array for field '{}'", + logical_field.name() + ); + }; + if logical_len != physical_len || source_list.value_length() != *physical_len { + return exec_err!( + "cannot adapt fixed-size list field '{}' from length {} to {}", + logical_field.name(), + source_list.value_length(), + physical_len + ); + } + let values = + adapt_array_to_physical_field_inner(source_list.values(), logical, physical, None)?; + Ok(Arc::new(FixedSizeListArray::new( + Arc::clone(physical), + *physical_len, + values, + source_list.nulls().cloned(), + ))) + } + (ArrowDataType::Map(logical, _), ArrowDataType::Map(physical, sorted)) => { + let Some(source_map) = source.as_any().downcast_ref::() else { + return exec_err!("expected map array for field '{}'", logical_field.name()); + }; + let (ArrowDataType::Struct(logical_entries), ArrowDataType::Struct(physical_entries)) = + (logical.data_type(), physical.data_type()) + else { + return exec_err!("map field '{}' has invalid entries", logical_field.name()); + }; + if logical_entries.len() != 2 || physical_entries.len() != 2 { + return exec_err!( + "cannot adapt map field '{}': expected key/value entries, got {} logical and {} physical entries", + logical_field.name(), + logical_entries.len(), + physical_entries.len() + ); + } + let arrays = physical_entries + .iter() + .map(|physical| { + if !matches!(physical.name().as_str(), "key" | "value") { + return exec_err!( + "map field '{}' has unexpected physical entry '{}'", + logical_field.name(), + physical.name() + ); + } + let logical = logical_entries + .iter() + .find(|logical| logical.name() == physical.name()) + .ok_or_else(|| { + DataFusionError::Plan(format!( + "map entry '{}' is missing from logical schema", + physical.name() + )) + })?; + let source = source_map + .entries() + .column_by_name(logical.name()) + .ok_or_else(|| { + DataFusionError::Plan(format!( + "map entry '{}' is missing from input", + logical.name() + )) + })?; + adapt_array_to_physical_field_inner( + source, + logical.as_ref(), + physical.as_ref(), + source_map.entries().nulls(), + ) + }) + .collect::>>()?; + let entries = StructArray::new(physical_entries.clone(), arrays, None); + Ok(Arc::new(MapArray::try_new( + Arc::clone(physical), + source_map.offsets().clone(), + entries, + source_map.nulls().cloned(), + *sorted, + )?)) + } + _ => cast_array_recursively(source, physical_field.data_type()), + } +} + fn make_physical_arrow_field(field: &Field, mode: ColumnMappingMode) -> Field { let physical_name_key = ColumnMetadataKey::ColumnMappingPhysicalName.as_ref(); let field_id_key = ColumnMetadataKey::ColumnMappingId.as_ref(); @@ -121,6 +460,52 @@ fn make_physical_arrow_data_type(dt: &ArrowDataType, mode: ColumnMappingMode) -> .collect(); ArrowDataType::Struct(new_fields) } + ArrowDataType::List(field) => ArrowDataType::List(Arc::new( + field + .as_ref() + .clone() + .with_data_type(make_physical_arrow_data_type(field.data_type(), mode)), + )), + ArrowDataType::LargeList(field) => ArrowDataType::LargeList(Arc::new( + field + .as_ref() + .clone() + .with_data_type(make_physical_arrow_data_type(field.data_type(), mode)), + )), + ArrowDataType::FixedSizeList(field, len) => ArrowDataType::FixedSizeList( + Arc::new( + field + .as_ref() + .clone() + .with_data_type(make_physical_arrow_data_type(field.data_type(), mode)), + ), + *len, + ), + ArrowDataType::Map(entries, sorted) => { + let ArrowDataType::Struct(fields) = entries.data_type() else { + return dt.clone(); + }; + let fields: Fields = fields + .iter() + .map(|field| { + Arc::new( + field + .as_ref() + .clone() + .with_data_type(make_physical_arrow_data_type(field.data_type(), mode)), + ) + }) + .collect(); + ArrowDataType::Map( + Arc::new( + entries + .as_ref() + .clone() + .with_data_type(ArrowDataType::Struct(fields)), + ), + *sorted, + ) + } other => other.clone(), } } @@ -182,157 +567,467 @@ fn wrap_partition_type(data_type: &ArrowDataType) -> ArrowDataType { } } -pub fn enrich_arrow_with_parquet_field_ids( +fn enrich_arrow_with_parquet_field_ids( physical_arrow: &ArrowSchema, logical_kernel: &StructType, -) -> ArrowSchema { - fn build_path_map( - st: &StructType, - path: &mut Vec, - out: &mut HashMap, (Option, String)>, - ) { - for f in st.fields() { - let phys = match f.get_config_value(&ColumnMetadataKey::ColumnMappingPhysicalName) { - Some(MetadataValue::String(s)) => s.clone(), - _ => f.name().clone(), - }; - path.push(phys); - let id = match f.get_config_value(&ColumnMetadataKey::ColumnMappingId) { - Some(MetadataValue::Number(fid)) => Some(*fid), - _ => None, - }; - out.insert(path.clone(), (id, f.name().clone())); - - match f.data_type() { - DataType::Struct(nst) => build_path_map(nst.as_ref(), path, out), - DataType::Array(at) => { - if let DataType::Struct(nst) = at.element_type() { - build_path_map(nst.as_ref(), path, out) - } +) -> DeltaResult { + fn enrich_data_type( + physical: &ArrowDataType, + logical: &DataType, + ) -> DeltaResult { + match (physical, logical) { + (ArrowDataType::Struct(physical_fields), DataType::Struct(logical_fields)) => { + let logical_fields = logical_fields.fields().collect::>(); + if physical_fields.len() != logical_fields.len() { + return Err(DeltaTableError::schema(format!( + "physical struct field count {} differs from logical field count {}", + physical_fields.len(), + logical_fields.len() + ))); } - DataType::Map(mt) => { - if let DataType::Struct(nst) = mt.value_type() { - build_path_map(nst.as_ref(), path, out) - } + Ok(ArrowDataType::Struct( + physical_fields + .iter() + .zip(logical_fields) + .map(|(physical, logical)| { + enrich_field(physical.as_ref(), logical).map(Arc::new) + }) + .collect::>()?, + )) + } + (ArrowDataType::List(physical), DataType::Array(logical)) => Ok(ArrowDataType::List( + Arc::new(physical.as_ref().clone().with_data_type(enrich_data_type( + physical.data_type(), + logical.element_type(), + )?)), + )), + (ArrowDataType::LargeList(physical), DataType::Array(logical)) => Ok( + ArrowDataType::LargeList(Arc::new(physical.as_ref().clone().with_data_type( + enrich_data_type(physical.data_type(), logical.element_type())?, + ))), + ), + (ArrowDataType::FixedSizeList(physical, len), DataType::Array(logical)) => { + Ok(ArrowDataType::FixedSizeList( + Arc::new(physical.as_ref().clone().with_data_type(enrich_data_type( + physical.data_type(), + logical.element_type(), + )?)), + *len, + )) + } + (ArrowDataType::Map(physical, sorted), DataType::Map(logical)) => { + let ArrowDataType::Struct(physical_entries) = physical.data_type() else { + return Err(DeltaTableError::schema( + "physical map entries must be a struct", + )); + }; + if physical_entries.len() != 2 { + return Err(DeltaTableError::schema(format!( + "physical map must contain key/value entries, got {}", + physical_entries.len() + ))); } - _ => {} + let entries = physical_entries + .iter() + .map(|entry| { + let logical_type = match entry.name().as_str() { + "key" => logical.key_type(), + "value" => logical.value_type(), + name => { + return Err(DeltaTableError::schema(format!( + "unexpected physical map entry '{name}'" + ))); + } + }; + Ok(Arc::new(entry.as_ref().clone().with_data_type( + enrich_data_type(entry.data_type(), logical_type)?, + ))) + }) + .collect::>()?; + Ok(ArrowDataType::Map( + Arc::new( + physical + .as_ref() + .clone() + .with_data_type(ArrowDataType::Struct(entries)), + ), + *sorted, + )) } - path.pop(); + _ => Ok(physical.clone()), } } - fn enrich_field( - af: &datafusion::arrow::datatypes::Field, - path: &mut Vec, - map: &HashMap, (Option, String)>, - rename_current: bool, - ) -> datafusion::arrow::datatypes::Field { - let mut meta = af.metadata().clone(); - let mut new_name = af.name().clone(); - if let Some((maybe_id, logical_name)) = map.get(path) { - if let Some(fid) = maybe_id { - meta.insert("PARQUET:field_id".to_string(), fid.to_string()); - } - if rename_current { - new_name = logical_name.clone(); - } + fn enrich_field(physical: &Field, logical: &StructField) -> DeltaResult { + let mut metadata = physical.metadata().clone(); + if let Some(MetadataValue::Number(id)) = + logical.get_config_value(&ColumnMetadataKey::ColumnMappingId) + { + metadata.insert(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string()); } + Ok(Field::new( + physical.name(), + enrich_data_type(physical.data_type(), logical.data_type())?, + physical.is_nullable(), + ) + .with_metadata(metadata)) + } - let new_dt = match af.data_type() { - datafusion::arrow::datatypes::DataType::Struct(children) => { - let new_children: Vec = children - .iter() - .map(|child| { - path.push(child.name().clone()); - let updated = enrich_field(child, path, map, true); - path.pop(); - updated - }) - .collect(); - datafusion::arrow::datatypes::DataType::Struct(new_children.into()) - } - datafusion::arrow::datatypes::DataType::List(elem) => { - let updated_elem = enrich_field(elem.as_ref(), path, map, false); - datafusion::arrow::datatypes::DataType::List(Arc::new(updated_elem)) - } - datafusion::arrow::datatypes::DataType::LargeList(elem) => { - let updated_elem = enrich_field(elem.as_ref(), path, map, false); - datafusion::arrow::datatypes::DataType::LargeList(Arc::new(updated_elem)) - } - datafusion::arrow::datatypes::DataType::FixedSizeList(elem, len) => { - let updated_elem = enrich_field(elem.as_ref(), path, map, false); - datafusion::arrow::datatypes::DataType::FixedSizeList(Arc::new(updated_elem), *len) - } - datafusion::arrow::datatypes::DataType::Map(kv, is_sorted) => { - let kv_struct = match kv.data_type() { - datafusion::arrow::datatypes::DataType::Struct(children) => children, - _ => { - return datafusion::arrow::datatypes::Field::new( - af.name(), - af.data_type().clone(), - af.is_nullable(), - ) - .with_metadata(meta) - } - }; - let mut new_kv_children: Vec = - Vec::with_capacity(2); - for child in kv_struct.iter() { - if child.name() == "value" { - let value_dt = match child.data_type() { - datafusion::arrow::datatypes::DataType::Struct(value_children) => { - let new_value_children: Vec = - value_children - .iter() - .map(|grandchild| { - path.push(grandchild.name().clone()); - let updated = enrich_field(grandchild, path, map, true); - path.pop(); - updated - }) - .collect(); - datafusion::arrow::datatypes::DataType::Struct( - new_value_children.into(), - ) - } - _ => child.data_type().clone(), - }; - let updated = datafusion::arrow::datatypes::Field::new( - "value", - value_dt, - child.is_nullable(), - ) - .with_metadata(child.metadata().clone()); - new_kv_children.push(updated); - } else { - new_kv_children.push(child.as_ref().clone()); - } - } - let new_kv = datafusion::arrow::datatypes::Field::new( - kv.name(), - datafusion::arrow::datatypes::DataType::Struct(new_kv_children.into()), - kv.is_nullable(), - ); - datafusion::arrow::datatypes::DataType::Map(Arc::new(new_kv), *is_sorted) - } - _ => af.data_type().clone(), + let logical_fields = logical_kernel.fields().collect::>(); + if physical_arrow.fields().len() != logical_fields.len() { + return Err(DeltaTableError::schema(format!( + "physical schema field count {} differs from logical field count {}", + physical_arrow.fields().len(), + logical_fields.len() + ))); + } + let new_fields = physical_arrow + .fields() + .iter() + .zip(logical_fields) + .map(|(physical, logical)| enrich_field(physical.as_ref(), logical)) + .collect::>>()?; + + Ok(ArrowSchema::new(new_fields)) +} + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::panic)] +mod tests { + use std::collections::HashMap; + + use datafusion::arrow::array::{Int64Array, NullArray}; + use datafusion::arrow::buffer::NullBuffer; + use datafusion::arrow::record_batch::RecordBatchOptions; + + use super::*; + use crate::spec::{ArrayType, MapType}; + + fn mapped_field(name: &str, physical_name: &str, id: i64, data_type: DataType) -> StructField { + StructField::new(name, data_type, true).with_metadata([ + ("delta.columnMapping.id", MetadataValue::Number(id)), + ( + "delta.columnMapping.physicalName", + MetadataValue::String(physical_name.to_string()), + ), + ]) + } + + #[test] + fn explicit_schema_inherits_nested_column_mapping_metadata() { + let physical_name_key = ColumnMetadataKey::ColumnMappingPhysicalName.as_ref(); + let table_value = Arc::new( + Field::new("value", ArrowDataType::Int64, true).with_metadata(HashMap::from([( + physical_name_key.to_string(), + "col-value".to_string(), + )])), + ); + let table_level = Arc::new( + Field::new( + "level", + ArrowDataType::Struct(vec![table_value].into()), + true, + ) + .with_metadata(HashMap::from([( + physical_name_key.to_string(), + "col-level".to_string(), + )])), + ); + let table = ArrowSchema::new(vec![ + Field::new( + "payload", + ArrowDataType::Struct(vec![table_level].into()), + true, + ) + .with_metadata(HashMap::from([( + physical_name_key.to_string(), + "col-payload".to_string(), + )])), + ]); + let requested = ArrowSchema::new(vec![Field::new( + "payload", + ArrowDataType::Struct( + vec![Arc::new(Field::new( + "level", + ArrowDataType::Struct( + vec![Arc::new(Field::new("value", ArrowDataType::Int64, true))].into(), + ), + true, + ))] + .into(), + ), + true, + )]); + + let attached = attach_column_mapping_metadata(&requested, &table); + let payload = attached.field_with_name("payload").expect("payload"); + assert_eq!( + payload + .metadata() + .get(physical_name_key) + .map(String::as_str), + Some("col-payload") + ); + let ArrowDataType::Struct(payload_fields) = payload.data_type() else { + panic!("payload must be struct"); + }; + let level = payload_fields.first().expect("level"); + assert_eq!( + level.metadata().get(physical_name_key).map(String::as_str), + Some("col-level") + ); + let ArrowDataType::Struct(level_fields) = level.data_type() else { + panic!("level must be struct"); + }; + assert_eq!( + level_fields[0] + .metadata() + .get(physical_name_key) + .map(String::as_str), + Some("col-value") + ); + } + + #[test] + fn physical_name_schema_enriches_nested_field_ids() { + let nested = StructType::try_new(vec![mapped_field("rate", "col-rate", 2, DataType::LONG)]) + .expect("nested schema"); + let logical = StructType::try_new(vec![mapped_field( + "details", + "col-details", + 1, + DataType::Struct(Box::new(nested)), + )]) + .expect("logical schema"); + + let physical = + get_physical_arrow_schema(&logical, ColumnMappingMode::Name).expect("physical schema"); + let details = physical + .field_with_name("col-details") + .expect("physical details field"); + let ArrowDataType::Struct(children) = details.data_type() else { + panic!("details must be a struct"); }; + let rate = children.first().expect("nested rate field"); - datafusion::arrow::datatypes::Field::new(&new_name, new_dt, af.is_nullable()) - .with_metadata(meta) + assert_eq!(rate.name(), "col-rate"); + assert_eq!( + rate.metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .map(String::as_str), + Some("2") + ); } - let mut path_to_info: HashMap, (Option, String)> = HashMap::new(); - build_path_map(logical_kernel, &mut Vec::new(), &mut path_to_info); + #[test] + fn physical_schema_recurses_through_arrays_and_maps() { + let event = + StructType::try_new(vec![mapped_field("code", "col-code", 2, DataType::STRING)]) + .expect("event schema"); + let amount = StructType::try_new(vec![mapped_field( + "amount", + "col-amount", + 4, + DataType::LONG, + )]) + .expect("amount schema"); + let logical = StructType::try_new(vec![ + mapped_field( + "events", + "col-events", + 1, + DataType::Array(Box::new(ArrayType::new( + DataType::Struct(Box::new(event)), + true, + ))), + ), + mapped_field( + "lookup", + "col-lookup", + 3, + DataType::Map(Box::new(MapType::new( + DataType::STRING, + DataType::Struct(Box::new(amount)), + true, + ))), + ), + ]) + .expect("logical schema"); - let new_fields: Vec = physical_arrow - .fields() - .iter() - .map(|af| { - let mut path = vec![af.name().clone()]; - enrich_field(af, &mut path, &path_to_info, false) - }) - .collect(); + let physical = + get_physical_arrow_schema(&logical, ColumnMappingMode::Name).expect("physical schema"); + let events = physical + .field_with_name("col-events") + .expect("events field"); + assert_eq!( + events.metadata().get(PARQUET_FIELD_ID_META_KEY), + Some(&"1".to_string()) + ); + let ArrowDataType::List(element) = events.data_type() else { + panic!("events must be a list"); + }; + let ArrowDataType::Struct(event_fields) = element.data_type() else { + panic!("events element must be a struct"); + }; + let code = event_fields.first().expect("code field"); + assert_eq!(code.name(), "col-code"); + assert_eq!( + code.metadata().get(PARQUET_FIELD_ID_META_KEY), + Some(&"2".to_string()) + ); + + let lookup = physical + .field_with_name("col-lookup") + .expect("lookup field"); + let ArrowDataType::Map(entries, _) = lookup.data_type() else { + panic!("lookup must be a map"); + }; + let ArrowDataType::Struct(entries) = entries.data_type() else { + panic!("map entries must be a struct"); + }; + let value = entries + .iter() + .find(|field| field.name() == "value") + .expect("map value"); + let ArrowDataType::Struct(value_fields) = value.data_type() else { + panic!("map value must be a struct"); + }; + let amount = value_fields.first().expect("amount field"); + assert_eq!(amount.name(), "col-amount"); + assert_eq!( + amount.metadata().get(PARQUET_FIELD_ID_META_KEY), + Some(&"4".to_string()) + ); + } + + #[test] + fn restore_missing_nullable_column_as_nulls() { + let batch = RecordBatch::try_new_with_options( + Arc::new(ArrowSchema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(2)), + ) + .expect("batch"); + let target = Arc::new(ArrowSchema::new(vec![Field::new( + "optional", + ArrowDataType::Int64, + true, + )])); + + let restored = restore_logical_record_batch(&batch, &target, ColumnMappingMode::Name) + .expect("restore"); + + assert_eq!(restored.column(0).null_count(), 2); + } + + #[test] + fn restore_missing_required_column_fails() { + let batch = RecordBatch::try_new_with_options( + Arc::new(ArrowSchema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(1)), + ) + .expect("batch"); + let target = Arc::new(ArrowSchema::new(vec![Field::new( + "required", + ArrowDataType::Int64, + false, + )])); - ArrowSchema::new(new_fields) + let error = restore_logical_record_batch(&batch, &target, ColumnMappingMode::Name) + .expect_err("missing required column must fail"); + + assert!( + error + .to_string() + .contains("missing required column 'required'") + ); + } + + #[test] + fn parquet_field_id_enrichment_rejects_field_count_mismatch() { + let physical = ArrowSchema::new(vec![Field::new("physical", ArrowDataType::Int64, true)]); + let logical = StructType::try_new([ + StructField::new("first", DataType::LONG, true), + StructField::new("second", DataType::LONG, true), + ]) + .expect("logical schema"); + + let error = enrich_arrow_with_parquet_field_ids(&physical, &logical) + .expect_err("field count mismatch must fail"); + + assert!(error.to_string().contains("field count")); + } + + #[test] + fn physical_adaptation_rejects_required_child_null_under_valid_parent() { + let logical_child = Arc::new(Field::new("child", ArrowDataType::Int64, true)); + let physical_child = Arc::new(Field::new("col-child", ArrowDataType::Int64, false)); + let source = Arc::new(StructArray::new( + vec![Arc::clone(&logical_child)].into(), + vec![Arc::new(Int64Array::from(vec![None]))], + None, + )) as ArrayRef; + let logical = Field::new( + "parent", + ArrowDataType::Struct(vec![logical_child].into()), + true, + ); + let physical = Field::new( + "col-parent", + ArrowDataType::Struct(vec![physical_child].into()), + true, + ); + + let error = adapt_array_to_physical_field(&source, &logical, &physical) + .expect_err("required child null must fail"); + + assert!( + error + .to_string() + .contains("required field 'child' contains null") + ); + } + + #[test] + fn physical_adaptation_rejects_all_null_required_field() { + let source = Arc::new(NullArray::new(1)) as ArrayRef; + let logical = Field::new("required", ArrowDataType::Int64, true); + let physical = Field::new("col-required", ArrowDataType::Int64, false); + + let error = adapt_array_to_physical_field(&source, &logical, &physical) + .expect_err("required field must reject nulls"); + + assert!( + error + .to_string() + .contains("required field 'required' contains null") + ); + } + + #[test] + fn physical_adaptation_allows_required_child_under_null_parent() { + let logical_child = Arc::new(Field::new("child", ArrowDataType::Int64, true)); + let physical_child = Arc::new(Field::new("col-child", ArrowDataType::Int64, false)); + let source = Arc::new(StructArray::new( + vec![Arc::clone(&logical_child)].into(), + vec![Arc::new(Int64Array::from(vec![None]))], + Some(NullBuffer::from(vec![false])), + )) as ArrayRef; + let logical = Field::new( + "parent", + ArrowDataType::Struct(vec![logical_child].into()), + true, + ); + let physical = Field::new( + "col-parent", + ArrowDataType::Struct(vec![physical_child].into()), + true, + ); + + let adapted = + adapt_array_to_physical_field(&source, &logical, &physical).expect("adaptation"); + + assert_eq!(adapted.null_count(), 1); + } } diff --git a/crates/sail-delta-lake/src/schema/manager.rs b/crates/sail-delta-lake/src/schema/manager.rs index 4b1f76834..1090aa31c 100644 --- a/crates/sail-delta-lake/src/schema/manager.rs +++ b/crates/sail-delta-lake/src/schema/manager.rs @@ -16,9 +16,9 @@ use sail_common_datafusion::catalog::CatalogTableColumnIdentity; use super::mapping::{annotate_new_fields_for_column_mapping, compute_max_column_id}; use crate::spec::{ - contains_timestampntz, contains_variant, ColumnMappingMode, ColumnMetadataKey, - DeltaError as DeltaTableError, DeltaResult, Metadata, MetadataValue, Protocol, StructField, - StructType, TableFeature, TableProperties, + ColumnMappingMode, ColumnMetadataKey, DeltaError as DeltaTableError, DeltaResult, Metadata, + MetadataValue, Protocol, StructField, StructType, TableFeature, TableProperties, + contains_timestampntz, contains_variant, }; /// Check if a Delta StructType schema contains any columns with generation expressions. diff --git a/crates/sail-delta-lake/src/schema/mapping.rs b/crates/sail-delta-lake/src/schema/mapping.rs index 2e7d3339a..fbdbdd6d7 100644 --- a/crates/sail-delta-lake/src/schema/mapping.rs +++ b/crates/sail-delta-lake/src/schema/mapping.rs @@ -13,7 +13,11 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicI64, Ordering}; -use crate::spec::{ArrayType, DataType, MapType, MetadataValue, StructField, StructType}; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +use crate::spec::{ + ArrayType, ColumnMetadataKey, DataType, MapType, MetadataValue, StructField, StructType, +}; /// Annotate a logical kernel schema with column mapping metadata (id + physicalName) /// using a sequential id assignment. Intended only for new table creation (name mode). @@ -47,58 +51,33 @@ pub fn annotate_new_fields_for_column_mapping( /// Compute the maximum `delta.columnMapping.id` present in a logical kernel schema. pub fn compute_max_column_id(schema: &StructType) -> i64 { - fn max_in_field(field: &StructField) -> i64 { - let mut max_id = field - .metadata() - .get("delta.columnMapping.id") - .and_then(|v| match v { - MetadataValue::Number(n) => Some(*n), - _ => None, - }) - .unwrap_or_default(); - - match field.data_type() { - DataType::Struct(st) => { - for f in st.fields() { - max_id = max_id.max(max_in_field(f)); - } + fn max_in_data_type(data_type: &DataType) -> i64 { + match data_type { + DataType::Struct(struct_type) => struct_type + .fields() + .map(max_in_field) + .max() + .unwrap_or_default(), + DataType::Array(array_type) => max_in_data_type(array_type.element_type()), + DataType::Map(map_type) => { + max_in_data_type(map_type.key_type()).max(max_in_data_type(map_type.value_type())) } - DataType::Array(at) => { - if let DataType::Struct(st) = at.element_type() { - for f in st.fields() { - max_id = max_id.max(max_in_field(f)); - } - } - } - DataType::Map(mt) => { - if let DataType::Struct(st) = mt.key_type() { - for f in st.fields() { - max_id = max_id.max(max_in_field(f)); - } - } - if let DataType::Struct(st) = mt.value_type() { - for f in st.fields() { - max_id = max_id.max(max_in_field(f)); - } - } - } - _ => {} + _ => 0, } - - max_id } - let mut max_id = 0i64; - for f in schema.fields() { - max_id = max_id.max(max_in_field(f)); + fn max_in_field(field: &StructField) -> i64 { + let field_id = column_mapping_id(field).unwrap_or_default(); + + field_id.max(max_in_data_type(field.data_type())) } - max_id + + schema.fields().map(max_in_field).max().unwrap_or_default() } fn column_mapping_id(field: &StructField) -> Option { field - .metadata() - .get("delta.columnMapping.id") + .get_config_value(&ColumnMetadataKey::ColumnMappingId) .and_then(|v| match v { MetadataValue::Number(n) => Some(*n), _ => None, @@ -107,25 +86,32 @@ fn column_mapping_id(field: &StructField) -> Option { fn column_mapping_physical_name(field: &StructField) -> Option<&str> { field - .metadata() - .get("delta.columnMapping.physicalName") + .get_config_value(&ColumnMetadataKey::ColumnMappingPhysicalName) .and_then(|v| match v { MetadataValue::String(s) => Some(s.as_str()), _ => None, }) } +fn strip_parquet_field_id_metadata(metadata: &mut HashMap) { + metadata.remove(PARQUET_FIELD_ID_META_KEY); + metadata.remove(ColumnMetadataKey::ParquetFieldId.as_ref()); +} + fn merge_metadata(prev: &StructField, new: &StructField) -> HashMap { let mut merged = prev.metadata().clone(); + strip_parquet_field_id_metadata(&mut merged); for (key, value) in new.metadata() { - let is_column_mapping_key = - key == "delta.columnMapping.id" || key == "delta.columnMapping.physicalName"; + let is_column_mapping_key = key == ColumnMetadataKey::ColumnMappingId.as_ref() + || key == ColumnMetadataKey::ColumnMappingPhysicalName.as_ref(); if is_column_mapping_key { // Preserve existing column mapping metadata if present; otherwise add it. merged.entry(key.clone()).or_insert_with(|| value.clone()); - } else { + } else if key != PARQUET_FIELD_ID_META_KEY + && key != ColumnMetadataKey::ParquetFieldId.as_ref() + { // For non-column-mapping keys, prefer the value from the new field (to keep user-added metadata). merged.insert(key.clone(), value.clone()); } @@ -137,10 +123,15 @@ fn merge_metadata(prev: &StructField, new: &StructField) -> HashMap StructField { let next_id = counter.fetch_add(1, Ordering::Relaxed); let physical_name = format!("col-{}", uuid::Uuid::new_v4()); - let annotated = field.clone().add_metadata([ - ("delta.columnMapping.id", MetadataValue::Number(next_id)), + let mut annotated = field.clone(); + strip_parquet_field_id_metadata(&mut annotated.metadata); + let annotated = annotated.add_metadata([ + ( + ColumnMetadataKey::ColumnMappingId.as_ref(), + MetadataValue::Number(next_id), + ), ( - "delta.columnMapping.physicalName", + ColumnMetadataKey::ColumnMappingPhysicalName.as_ref(), MetadataValue::String(physical_name), ), ]); @@ -217,24 +208,22 @@ fn find_matching_field<'a>( candidate: &StructField, existing_fields: &'a [&StructField], ) -> Option<&'a StructField> { - if let Some(cid) = column_mapping_id(candidate) { - if let Some(field) = existing_fields + if let Some(cid) = column_mapping_id(candidate) + && let Some(field) = existing_fields .iter() .copied() .find(|f| column_mapping_id(f) == Some(cid)) - { - return Some(field); - } + { + return Some(field); } - if let Some(phys) = column_mapping_physical_name(candidate) { - if let Some(field) = existing_fields + if let Some(phys) = column_mapping_physical_name(candidate) + && let Some(field) = existing_fields .iter() .copied() .find(|f| column_mapping_physical_name(f) == Some(phys)) - { - return Some(field); - } + { + return Some(field); } existing_fields @@ -271,3 +260,41 @@ fn empty_struct_type() -> StructType { StructType::try_new(Vec::::new()) .unwrap_or_else(|_| unreachable!("empty struct type is always valid")) } + +#[cfg(test)] +#[expect(clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + + #[test] + fn annotation_strips_transient_parquet_field_ids() { + let rate = StructField::new("rate", DataType::LONG, true).with_metadata([ + (PARQUET_FIELD_ID_META_KEY, MetadataValue::Number(30)), + ( + ColumnMetadataKey::ParquetFieldId.as_ref(), + MetadataValue::Number(30), + ), + ]); + let nested = StructType::try_new(vec![rate]).expect("nested schema"); + let details = StructField::new("details", DataType::Struct(Box::new(nested)), true) + .with_metadata([(PARQUET_FIELD_ID_META_KEY, MetadataValue::Number(20))]); + let schema = StructType::try_new(vec![details]).expect("schema"); + + let annotated = annotate_schema_for_column_mapping(&schema); + let details = annotated.field("details").expect("details field"); + let DataType::Struct(nested) = details.data_type() else { + panic!("details must be a struct"); + }; + let rate = nested.field("rate").expect("rate field"); + + for field in [details, rate] { + assert!(!field.metadata().contains_key(PARQUET_FIELD_ID_META_KEY)); + assert!( + !field + .metadata() + .contains_key(ColumnMetadataKey::ParquetFieldId.as_ref()) + ); + assert!(field.metadata().contains_key("delta.columnMapping.id")); + } + } +} diff --git a/crates/sail-delta-lake/src/schema/mod.rs b/crates/sail-delta-lake/src/schema/mod.rs index 71383b590..f226c6910 100644 --- a/crates/sail-delta-lake/src/schema/mod.rs +++ b/crates/sail-delta-lake/src/schema/mod.rs @@ -17,9 +17,11 @@ pub mod mapping; pub mod normalize; pub mod type_widening; +pub(crate) use converter::adapt_array_to_physical_field; pub use converter::{ arrow_field_physical_name, arrow_schema_from_struct_type, arrow_schema_reorder_partitions, - get_physical_arrow_schema as get_physical_schema, make_physical_arrow_schema, + attach_column_mapping_metadata, get_physical_arrow_schema as get_physical_schema, + make_physical_arrow_schema, restore_logical_record_batch, }; pub use manager::{ evolve_schema, metadata_for_create_with_struct_type, protocol_for_create, @@ -33,7 +35,7 @@ pub use mapping::{ annotate_new_fields_for_column_mapping, annotate_schema_for_column_mapping, annotate_schema_for_column_mapping as annotate_for_column_mapping, compute_max_column_id, }; -pub use normalize::normalize_delta_schema; +pub use normalize::{normalize_delta_schema, strip_column_mapping_metadata}; pub use type_widening::{ add_type_widening_metadata, collect_type_changes, format_type_change_path, is_supported_type_change_for_schema_evolution, is_supported_type_change_for_write, diff --git a/crates/sail-delta-lake/src/schema/normalize.rs b/crates/sail-delta-lake/src/schema/normalize.rs index 53a01fd57..c69f7d3b2 100644 --- a/crates/sail-delta-lake/src/schema/normalize.rs +++ b/crates/sail-delta-lake/src/schema/normalize.rs @@ -1,37 +1,112 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef, TimeUnit}; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use sail_common_datafusion::column_features::SAIL_WRITE_TARGET_NULLABLE_METADATA_KEY; +use crate::spec::ColumnMetadataKey; + +const COLUMN_MAPPING_METADATA_PREFIX: &str = "delta.columnMapping."; +const PARQUET_FIELD_NESTED_IDS_METADATA_KEY: &str = "parquet.field.nested.ids"; + /// Normalize Arrow schemas for Delta Lake compatibility by rewriting timestamp /// fields to UTC microseconds. pub fn normalize_delta_schema(schema: &SchemaRef) -> SchemaRef { - let mut changed = false; let normalized_fields: Vec = schema .fields() .iter() - .map(|field_arc| normalize_field(field_arc.as_ref(), &mut changed)) + .map(|field| normalize_field(field.as_ref())) .collect(); - if changed { + if fields_differ(schema, &normalized_fields) { Arc::new(Schema::new(normalized_fields)) } else { - schema.clone() + Arc::clone(schema) } } -fn normalize_field(field: &Field, changed: &mut bool) -> Field { - let normalized_type = normalize_datatype(field.data_type()); +/// Remove source-table column mapping metadata before deriving a Delta target schema. +pub fn strip_column_mapping_metadata(schema: &SchemaRef) -> SchemaRef { + let stripped_fields = schema + .fields() + .iter() + .map(|field| strip_column_mapping_field(field.as_ref())) + .collect::>(); + + if fields_differ(schema, &stripped_fields) { + Arc::new(Schema::new_with_metadata( + stripped_fields, + schema.metadata().clone(), + )) + } else { + Arc::clone(schema) + } +} + +fn fields_differ(schema: &Schema, fields: &[Field]) -> bool { + !schema + .fields() + .iter() + .map(|field| field.as_ref()) + .eq(fields.iter()) +} + +fn strip_column_mapping_field(field: &Field) -> Field { + let stripped_type = transform_nested_fields(field.data_type(), strip_column_mapping_field); + let mut stripped_field = if &stripped_type != field.data_type() { + field.clone().with_data_type(stripped_type) + } else { + field.clone() + }; + let mut metadata = stripped_field.metadata().clone(); + let previous_len = metadata.len(); + metadata.retain(|key, _| { + !key.starts_with(COLUMN_MAPPING_METADATA_PREFIX) + && key != PARQUET_FIELD_ID_META_KEY + && key != ColumnMetadataKey::ParquetFieldId.as_ref() + && key != PARQUET_FIELD_NESTED_IDS_METADATA_KEY + }); + if metadata.len() != previous_len { + stripped_field = stripped_field.with_metadata(metadata); + } + stripped_field +} + +fn transform_nested_fields(data_type: &DataType, transform: fn(&Field) -> Field) -> DataType { + match data_type { + DataType::Struct(fields) => { + let transformed_fields = fields + .iter() + .map(|field| Arc::new(transform(field.as_ref()))) + .collect::(); + DataType::Struct(transformed_fields) + } + DataType::List(field) => DataType::List(Arc::new(transform(field.as_ref()))), + DataType::LargeList(field) => DataType::LargeList(Arc::new(transform(field.as_ref()))), + DataType::FixedSizeList(field, size) => { + DataType::FixedSizeList(Arc::new(transform(field.as_ref())), *size) + } + DataType::Map(field, sorted) => DataType::Map(Arc::new(transform(field.as_ref())), *sorted), + _ => data_type.clone(), + } +} + +fn normalize_field(field: &Field) -> Field { + let normalized_type = match field.data_type() { + DataType::Timestamp(_, Some(_)) => { + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::::from("UTC"))) + } + data_type => transform_nested_fields(data_type, normalize_field), + }; let normalized_field = if &normalized_type != field.data_type() { - *changed = true; field.clone().with_data_type(normalized_type) } else { field.clone() }; - strip_sail_write_metadata(normalized_field, changed) + strip_sail_write_metadata(normalized_field) } -fn strip_sail_write_metadata(mut field: Field, changed: &mut bool) -> Field { +fn strip_sail_write_metadata(mut field: Field) -> Field { if field .metadata() .contains_key(SAIL_WRITE_TARGET_NULLABLE_METADATA_KEY) @@ -39,64 +114,6 @@ fn strip_sail_write_metadata(mut field: Field, changed: &mut bool) -> Field { let mut metadata = field.metadata().clone(); metadata.remove(SAIL_WRITE_TARGET_NULLABLE_METADATA_KEY); field = field.with_metadata(metadata); - *changed = true; } field } - -fn normalize_datatype(data_type: &DataType) -> DataType { - match data_type { - DataType::Timestamp(_, Some(_)) => { - DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::::from("UTC"))) - } - DataType::Struct(fields) => { - let mut changed = false; - let normalized_fields: Fields = fields - .iter() - .map(|field_arc| Arc::new(normalize_field(field_arc.as_ref(), &mut changed))) - .collect(); - if changed { - DataType::Struct(normalized_fields) - } else { - data_type.clone() - } - } - DataType::List(field) => { - let mut changed = false; - let normalized_field = normalize_field(field.as_ref(), &mut changed); - if changed { - DataType::List(Arc::new(normalized_field)) - } else { - data_type.clone() - } - } - DataType::LargeList(field) => { - let mut changed = false; - let normalized_field = normalize_field(field.as_ref(), &mut changed); - if changed { - DataType::LargeList(Arc::new(normalized_field)) - } else { - data_type.clone() - } - } - DataType::FixedSizeList(field, size) => { - let mut changed = false; - let normalized_field = normalize_field(field.as_ref(), &mut changed); - if changed { - DataType::FixedSizeList(Arc::new(normalized_field), *size) - } else { - data_type.clone() - } - } - DataType::Map(field, sorted) => { - let mut changed = false; - let normalized_field = normalize_field(field.as_ref(), &mut changed); - if changed { - DataType::Map(Arc::new(normalized_field), *sorted) - } else { - data_type.clone() - } - } - _ => data_type.clone(), - } -} diff --git a/crates/sail-delta-lake/src/schema/type_widening.rs b/crates/sail-delta-lake/src/schema/type_widening.rs index cc02b4fa1..e126540d3 100644 --- a/crates/sail-delta-lake/src/schema/type_widening.rs +++ b/crates/sail-delta-lake/src/schema/type_widening.rs @@ -700,17 +700,17 @@ fn resolve_field_path<'a>( (DataType::Array(_), other) => { return Err(DeltaTableError::schema(format!( "expected 'element' for array type change path, found '{other}'" - ))) + ))); } (DataType::Map(_), other) => { return Err(DeltaTableError::schema(format!( "expected 'key' or 'value' for map type change path, found '{other}'" - ))) + ))); } (other_type, other) => { return Err(DeltaTableError::schema(format!( "cannot resolve type change path segment '{other}' through {other_type}" - ))) + ))); } } } diff --git a/crates/sail-delta-lake/src/session_extension.rs b/crates/sail-delta-lake/src/session_extension.rs index fde0642e7..f35932fd8 100644 --- a/crates/sail-delta-lake/src/session_extension.rs +++ b/crates/sail-delta-lake/src/session_extension.rs @@ -6,11 +6,11 @@ use moka::future::Cache as FutureCache; use sail_common_datafusion::catalog::LakehouseExecutionContext; use url::Url; -use crate::delta_log::StorageConfig; +use crate::delta_log::{LogStoreRef, StorageConfig}; use crate::snapshot::DeltaSnapshotConfig; use crate::table::{ - create_logstore_with_object_store, load_catalog_managed_commits_for_snapshot, DeltaSnapshot, - DeltaTable, + DeltaSnapshot, DeltaTable, catalog_managed_commit_context, create_logstore_with_object_store, + load_catalog_managed_commits_for_snapshot, }; const DEFAULT_MAX_ENTRIES: u64 = 1024; @@ -58,6 +58,7 @@ impl DeltaTableCache { version: i64, lakehouse_table: Option<&LakehouseExecutionContext>, ) -> Result> { + let lakehouse_table = catalog_managed_commit_context(lakehouse_table); let key = TableCacheKey::new(table_url, version, lakehouse_table); let table_url = table_url.clone(); let lakehouse_table = lakehouse_table.cloned(); @@ -100,7 +101,7 @@ pub(crate) async fn load_table_uncached( require_files: false, ..Default::default() }; - if let Some(lakehouse_table) = lakehouse_table { + if let Some(lakehouse_table) = catalog_managed_commit_context(lakehouse_table) { table_config.catalog_managed_commits = load_catalog_managed_commits_for_snapshot( context, lakehouse_table, @@ -110,6 +111,31 @@ pub(crate) async fn load_table_uncached( ) .await?; } + load_table_from_log_store(version, log_store, table_config).await +} + +pub(crate) async fn load_table_with_config( + context: &TaskContext, + table_url: &Url, + version: i64, + table_config: DeltaSnapshotConfig, +) -> Result> { + let object_store = context + .runtime_env() + .object_store_registry + .get_store(table_url) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let log_store = + create_logstore_with_object_store(object_store, table_url.clone(), StorageConfig) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + load_table_from_log_store(version, log_store, table_config).await +} + +async fn load_table_from_log_store( + version: i64, + log_store: LogStoreRef, + table_config: DeltaSnapshotConfig, +) -> Result> { let mut table = DeltaTable::new(log_store.clone(), table_config); table .load_version(version) diff --git a/crates/sail-delta-lake/src/snapshot/config.rs b/crates/sail-delta-lake/src/snapshot/config.rs index c5bf3ce08..a2b8f8df8 100644 --- a/crates/sail-delta-lake/src/snapshot/config.rs +++ b/crates/sail-delta-lake/src/snapshot/config.rs @@ -19,17 +19,18 @@ // [Credit]: use object_store::path::Path; +use serde::{Deserialize, Serialize}; /// Configuration options for loading Delta table snapshots. /// /// Controls how the log-replay reads the `_delta_log`. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CatalogManagedCommitFile { pub version: i64, pub file_name: String, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CatalogManagedCommitSet { pub latest_table_version: i64, pub commits: Vec, diff --git a/crates/sail-delta-lake/src/snapshot/materialize.rs b/crates/sail-delta-lake/src/snapshot/materialize.rs index 87da3bd06..6d3215aba 100644 --- a/crates/sail-delta-lake/src/snapshot/materialize.rs +++ b/crates/sail-delta-lake/src/snapshot/materialize.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use datafusion::arrow::array::{ - new_empty_array, Array, ArrayRef, MapArray, StringArray, StructArray, + Array, ArrayRef, MapArray, StringArray, StructArray, new_empty_array, }; use datafusion::arrow::datatypes::{Field, Fields, Schema as ArrowSchema}; use datafusion::arrow::record_batch::RecordBatch; @@ -15,8 +15,8 @@ use crate::spec::fields::{ FIELD_NAME_PARTITION_VALUES_PARSED, FIELD_NAME_STATS, FIELD_NAME_STATS_PARSED, }; use crate::spec::{ - parse_stats_json_array, stats_schema, Add, ColumnMappingMode, DataType, - DeltaError as DeltaTableError, DeltaResult, StructType, + Add, ColumnMappingMode, DataType, DeltaError as DeltaTableError, DeltaResult, StructType, + parse_stats_json_array, stats_schema, }; impl DeltaSnapshot { @@ -189,7 +189,10 @@ pub(crate) fn parse_partition_values_array( let arrow_fields: Fields = Fields::from( partition_schema .fields() - .map(|field| Field::try_from(&field.make_physical(column_mapping_mode))) + .map(|field| { + Field::try_from(&field.make_physical(column_mapping_mode)) + .map(|field| field.with_nullable(true)) + }) .collect::, _>>()?, ); @@ -226,7 +229,11 @@ pub(crate) fn parse_partition_values_array( }) .collect::>>()?; - Ok(StructArray::try_new(arrow_fields, columns, None)?) + Ok(StructArray::try_new( + arrow_fields, + columns, + partitions.nulls().cloned(), + )?) } fn map_array_from_path<'a>(batch: &'a RecordBatch, path: &str) -> DeltaResult<&'a MapArray> { diff --git a/crates/sail-delta-lake/src/snapshot/mod.rs b/crates/sail-delta-lake/src/snapshot/mod.rs index 36e24ecb3..18b528232 100644 --- a/crates/sail-delta-lake/src/snapshot/mod.rs +++ b/crates/sail-delta-lake/src/snapshot/mod.rs @@ -33,11 +33,11 @@ use serde_json::Value; use url::Url; use crate::checkpoint::{ - latest_replayable_version, load_replayed_table_header, load_replayed_table_state, - ReplayedTableState, + ReplayedTableState, latest_replayable_version, load_replayed_table_header, + load_replayed_table_state, }; -use crate::delta_log::segment_files::ReplayedTableHeader; use crate::delta_log::LogStore; +use crate::delta_log::segment_files::ReplayedTableHeader; use crate::schema::{ arrow_field_physical_name, arrow_schema_reorder_partitions, protocol_supports_type_widening, schema_contains_type_widening_metadata, validate_type_widening_metadata, @@ -64,8 +64,8 @@ mod stats; pub use config::DeltaSnapshotConfig; pub(crate) use config::{ - catalog_managed_commit_file_name, catalog_managed_commit_path, CatalogManagedCommitFile, - CatalogManagedCommitSet, + CatalogManagedCommitFile, CatalogManagedCommitSet, catalog_managed_commit_file_name, + catalog_managed_commit_path, }; pub struct DeltaSnapshot { @@ -172,7 +172,7 @@ impl DeltaSnapshot { Err(crate::spec::DeltaError::MissingVersion) => { return Err(DeltaTableError::invalid_table_location( "No commit files found in _delta_log", - )) + )); } Err(err) => return Err(err), } @@ -190,7 +190,7 @@ impl DeltaSnapshot { .await { Ok(Some(replayed)) => { - return Self::from_replayed_header(log_store, config, replayed) + return Self::from_replayed_header(log_store, config, replayed); } Ok(None) => {} Err(err) => { @@ -602,7 +602,7 @@ impl DeltaSnapshot { pub async fn all_tombstones( &self, log_store: &dyn LogStore, - ) -> DeltaResult> { + ) -> DeltaResult + use<>> { Ok(self .tombstones(log_store) .try_collect::>() @@ -613,7 +613,7 @@ impl DeltaSnapshot { pub async fn unexpired_tombstones( &self, log_store: &dyn LogStore, - ) -> DeltaResult> { + ) -> DeltaResult + use<>> { let retention_timestamp = Utc::now().timestamp_millis() - self .table_properties() @@ -1001,14 +1001,14 @@ mod tests { use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; - use object_store::memory::InMemory; use object_store::ObjectStore; + use object_store::memory::InMemory; use once_cell::sync::OnceCell; use url::Url; use super::DeltaSnapshot; use crate::datasource::DeltaScanConfig; - use crate::delta_log::{default_logstore, LogStoreRef, StorageConfig}; + use crate::delta_log::{LogStoreRef, StorageConfig, default_logstore}; use crate::logical::table_source::DeltaTableSource; use crate::snapshot::{CatalogManagedCommitSet, DeltaSnapshotConfig}; use crate::spec::{ @@ -1429,9 +1429,11 @@ mod tests { let protocol = Protocol::new(1, 7, None, Some(vec![TableFeature::VacuumProtocolCheck])); let snapshot = test_snapshot(protocol, test_metadata([]), Vec::new()); - assert!(snapshot - .build_version_checksum(None, None) - .unwrap() - .is_none()); + assert!( + snapshot + .build_version_checksum(None, None) + .unwrap() + .is_none() + ); } } diff --git a/crates/sail-delta-lake/src/spec/actions.rs b/crates/sail-delta-lake/src/spec/actions.rs index f58f26c6a..3aea763d8 100644 --- a/crates/sail-delta-lake/src/spec/actions.rs +++ b/crates/sail-delta-lake/src/spec/actions.rs @@ -23,8 +23,8 @@ use std::hash::{Hash, Hasher}; use std::str::FromStr; use chrono::DateTime; -use object_store::path::Path; use object_store::ObjectMeta; +use object_store::path::Path; use serde::{Deserialize, Serialize}; use crate::spec::statistics::Stats; @@ -436,11 +436,9 @@ impl TryFrom for CommitAction { Action::Txn(t) => Ok(Self::Txn(t)), Action::CommitInfo(c) => Ok(Self::CommitInfo(c)), Action::DomainMetadata(d) => Ok(Self::DomainMetadata(d)), - Action::CheckpointMetadata(_) | Action::Sidecar(_) => Err( - DeltaTableError::generic( - "checkpoint-only actions (CheckpointMetadata, Sidecar) are not allowed in commit files", - ), - ), + Action::CheckpointMetadata(_) | Action::Sidecar(_) => Err(DeltaTableError::generic( + "checkpoint-only actions (CheckpointMetadata, Sidecar) are not allowed in commit files", + )), } } } diff --git a/crates/sail-delta-lake/src/spec/checkpoint.rs b/crates/sail-delta-lake/src/spec/checkpoint.rs index 865fd1c35..365c598e7 100644 --- a/crates/sail-delta-lake/src/spec/checkpoint.rs +++ b/crates/sail-delta-lake/src/spec/checkpoint.rs @@ -21,10 +21,10 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use crate::spec::{ - add_struct_type, checkpoint_metadata_struct_type, domain_metadata_struct_type, - metadata_struct_type, protocol_struct_type, remove_struct_type, sidecar_struct_type, - transaction_struct_type, Add, CheckpointMetadata, DataType, DomainMetadata, Metadata, Protocol, - Remove, Sidecar, StructField, StructType, Transaction, + Add, CheckpointMetadata, DataType, DomainMetadata, Metadata, Protocol, Remove, Sidecar, + StructField, StructType, Transaction, add_struct_type, checkpoint_metadata_struct_type, + domain_metadata_struct_type, metadata_struct_type, protocol_struct_type, remove_struct_type, + sidecar_struct_type, transaction_struct_type, }; // [Credit]: diff --git a/crates/sail-delta-lake/src/spec/error.rs b/crates/sail-delta-lake/src/spec/error.rs index 7244964ab..8100c2d70 100644 --- a/crates/sail-delta-lake/src/spec/error.rs +++ b/crates/sail-delta-lake/src/spec/error.rs @@ -30,13 +30,19 @@ pub type DeltaResult = Result; // [Credit]: #[derive(Error, Debug)] pub enum CommitConflictError { - #[error("Commit failed: a concurrent transactions added new data.\nHelp: This transaction's query must be rerun to include the new data. Also, if you don't care to require this check to pass in the future, the isolation level can be set to Snapshot Isolation.")] + #[error( + "Commit failed: a concurrent transactions added new data.\nHelp: This transaction's query must be rerun to include the new data. Also, if you don't care to require this check to pass in the future, the isolation level can be set to Snapshot Isolation." + )] ConcurrentAppend, - #[error("Commit failed: a concurrent transaction deleted data this operation read.\nHelp: This transaction's query must be rerun to exclude the removed data. Also, if you don't care to require this check to pass in the future, the isolation level can be set to Snapshot Isolation.")] + #[error( + "Commit failed: a concurrent transaction deleted data this operation read.\nHelp: This transaction's query must be rerun to exclude the removed data. Also, if you don't care to require this check to pass in the future, the isolation level can be set to Snapshot Isolation." + )] ConcurrentDeleteRead, - #[error("Commit failed: a concurrent transaction deleted the same data your transaction deletes.\nHelp: you should retry this write operation. If it was based on data contained in the table, you should rerun the query generating the data.")] + #[error( + "Commit failed: a concurrent transaction deleted the same data your transaction deletes.\nHelp: you should retry this write operation. If it was based on data contained in the table, you should rerun the query generating the data." + )] ConcurrentDeleteDelete, #[error("Metadata changed since last commit.")] diff --git a/crates/sail-delta-lake/src/spec/log.rs b/crates/sail-delta-lake/src/spec/log.rs index cf458e995..0b641034b 100644 --- a/crates/sail-delta-lake/src/spec/log.rs +++ b/crates/sail-delta-lake/src/spec/log.rs @@ -16,7 +16,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use object_store::path::{Path, DELIMITER}; +use object_store::path::{DELIMITER, Path}; use uuid::Uuid; // [Credit]: diff --git a/crates/sail-delta-lake/src/spec/mod.rs b/crates/sail-delta-lake/src/spec/mod.rs index 61612d42a..825c8b99d 100644 --- a/crates/sail-delta-lake/src/spec/mod.rs +++ b/crates/sail-delta-lake/src/spec/mod.rs @@ -17,37 +17,37 @@ pub use action_schema::{ domain_metadata_struct_type, metadata_struct_type, protocol_struct_type, remove_struct_type, sidecar_struct_type, transaction_struct_type, }; -pub(crate) use actions::{logical_file_key, LogicalFileKey}; pub use actions::{ Action, Add, CheckpointMetadata, CommitAction, CommitInfo, DeletionVectorDescriptor, DomainMetadata, Remove, RemoveOptions, Sidecar, StorageType, Transaction, }; +pub(crate) use actions::{LogicalFileKey, logical_file_key}; pub use checkpoint::{CheckpointActionRow, LastCheckpointHint}; pub use checksum::VersionChecksum; pub use datafusion::arrow::datatypes::SchemaRef; pub use error::{CommitConflictError, DeltaError, DeltaResult, TransactionError}; pub use log::{ - checkpoint_path, checksum_path, commit_path, compacted_json_path, delta_log_file_path, - delta_log_prefix_path, delta_log_root_path, is_compacted_json_filename, - is_json_checkpoint_filename, is_uuid_checkpoint_filename, last_checkpoint_path, - parse_checkpoint_version, parse_checksum_version, parse_commit_version, + DELTA_LOG_DIR, LAST_CHECKPOINT_FILE, SIDECARS_DIR, checkpoint_path, checksum_path, commit_path, + compacted_json_path, delta_log_file_path, delta_log_prefix_path, delta_log_root_path, + is_compacted_json_filename, is_json_checkpoint_filename, is_uuid_checkpoint_filename, + last_checkpoint_path, parse_checkpoint_version, parse_checksum_version, parse_commit_version, parse_compacted_json_versions, parse_version_prefix, sidecar_file_name, sidecar_file_path, sidecar_log_path, sidecars_dir_path, staged_commit_path, temp_commit_path, - uuid_checkpoint_path, DELTA_LOG_DIR, LAST_CHECKPOINT_FILE, SIDECARS_DIR, + uuid_checkpoint_path, }; pub use metadata::{Format, Metadata}; pub use operation::{DeltaOperation, MergePredicate, SaveMode}; pub use properties::{ - canonicalize_and_validate_table_properties, route_table_property_key, DataSkippingNumIndexedCols, IsolationLevel, TableProperties, + canonicalize_and_validate_table_properties, route_table_property_key, }; pub use protocol::{Protocol, TableFeature}; pub use schema::{ ArrayType, ColumnMappingMode, ColumnMetadataKey, ColumnName, DataType, DecimalType, MapType, MetadataValue, PrimitiveType, Schema, StructField, StructType, }; -pub(crate) use statistics::{parse_stats_json_array, stats_schema}; pub use statistics::{ColumnCountStat, ColumnValueStat, MaxStat, MinStat, StatValue, Stats}; +pub(crate) use statistics::{parse_stats_json_array, stats_schema}; pub(crate) use utils::{ contains_timestampntz, contains_timestampntz_arrow, contains_variant, contains_variant_arrow, }; diff --git a/crates/sail-delta-lake/src/spec/properties.rs b/crates/sail-delta-lake/src/spec/properties.rs index 08e781af2..582ce3a54 100644 --- a/crates/sail-delta-lake/src/spec/properties.rs +++ b/crates/sail-delta-lake/src/spec/properties.rs @@ -429,8 +429,7 @@ fn parse_non_negative_i64(s: &str) -> Option { fn parse_interval(s: &str) -> Option { // [Credit]: - const SECONDS_PER_MINUTE: u64 = 60; - const SECONDS_PER_HOUR: u64 = 60 * SECONDS_PER_MINUTE; + const SECONDS_PER_HOUR: u64 = 60 * 60; const SECONDS_PER_DAY: u64 = 24 * SECONDS_PER_HOUR; const SECONDS_PER_WEEK: u64 = 7 * SECONDS_PER_DAY; @@ -448,10 +447,14 @@ fn parse_interval(s: &str) -> Option { "microsecond" | "microseconds" => Some(Duration::from_micros(number)), "millisecond" | "milliseconds" => Some(Duration::from_millis(number)), "second" | "seconds" => Some(Duration::from_secs(number)), - "minute" | "minutes" => Some(Duration::from_secs(number * SECONDS_PER_MINUTE)), - "hour" | "hours" => Some(Duration::from_secs(number * SECONDS_PER_HOUR)), - "day" | "days" => Some(Duration::from_secs(number * SECONDS_PER_DAY)), - "week" | "weeks" => Some(Duration::from_secs(number * SECONDS_PER_WEEK)), + "minute" | "minutes" => (number <= u64::MAX / 60).then(|| Duration::from_mins(number)), + "hour" | "hours" => { + (number <= u64::MAX / SECONDS_PER_HOUR).then(|| Duration::from_hours(number)) + } + "day" | "days" => number.checked_mul(SECONDS_PER_DAY).map(Duration::from_secs), + "week" | "weeks" => number + .checked_mul(SECONDS_PER_WEEK) + .map(Duration::from_secs), _ => None, } } @@ -525,9 +528,10 @@ mod tests { canonicalize_and_validate_table_properties([("delta.checkpointInterval", "0")]); assert!(result.is_err()); if let Err(err) = result { - assert!(err - .to_string() - .contains("invalid value for delta.checkpointInterval")); + assert!( + err.to_string() + .contains("invalid value for delta.checkpointInterval") + ); } } diff --git a/crates/sail-delta-lake/src/spec/schema.rs b/crates/sail-delta-lake/src/spec/schema.rs index 197be1cef..21d6b46bd 100644 --- a/crates/sail-delta-lake/src/spec/schema.rs +++ b/crates/sail-delta-lake/src/spec/schema.rs @@ -460,10 +460,7 @@ impl Display for StructField { } fn make_physical_field(field: &StructField, column_mapping_mode: ColumnMappingMode) -> StructField { - let data_type = match &field.data_type { - DataType::Struct(inner) => DataType::from(inner.make_physical(column_mapping_mode)), - other => other.clone(), - }; + let data_type = make_physical_data_type(&field.data_type, column_mapping_mode); let mut metadata = field.metadata().clone(); let physical_name_key = ColumnMetadataKey::ColumnMappingPhysicalName.as_ref(); @@ -503,6 +500,25 @@ fn make_physical_field(field: &StructField, column_mapping_mode: ColumnMappingMo } } +fn make_physical_data_type( + data_type: &DataType, + column_mapping_mode: ColumnMappingMode, +) -> DataType { + match data_type { + DataType::Struct(inner) => DataType::from(inner.make_physical(column_mapping_mode)), + DataType::Array(array) => DataType::Array(Box::new(ArrayType::new( + make_physical_data_type(array.element_type(), column_mapping_mode), + array.contains_null(), + ))), + DataType::Map(map) => DataType::Map(Box::new(MapType::new( + make_physical_data_type(map.key_type(), column_mapping_mode), + make_physical_data_type(map.value_type(), column_mapping_mode), + map.value_contains_null(), + ))), + other => other.clone(), + } +} + #[derive(Debug, PartialEq, Clone, Eq)] // [Credit]: pub struct StructType { diff --git a/crates/sail-delta-lake/src/spec/statistics.rs b/crates/sail-delta-lake/src/spec/statistics.rs index 1fec7b0e7..e5f5168ae 100644 --- a/crates/sail-delta-lake/src/spec/statistics.rs +++ b/crates/sail-delta-lake/src/spec/statistics.rs @@ -368,11 +368,8 @@ fn null_count_stats_schema(schema: &StructType) -> Option { let data_type = match &field.data_type { DataType::Array(_) | DataType::Map(_) | DataType::Variant(_) => DataType::LONG, DataType::Struct(inner) => { - if let Some(inner_schema) = null_count_stats_schema(inner) { - DataType::from(inner_schema) - } else { - return None; - } + let inner_schema = null_count_stats_schema(inner)?; + DataType::from(inner_schema) } DataType::Primitive(_) => DataType::LONG, }; @@ -429,10 +426,10 @@ fn base_stats_schema_fields( ) -> Vec { let mut result = Vec::new(); for field in schema.fields() { - if let Some(DataSkippingNumIndexedCols::NumColumns(n_cols)) = n_columns { - if *added_columns >= *n_cols { - break; - } + if let Some(DataSkippingNumIndexedCols::NumColumns(n_cols)) = n_columns + && *added_columns >= *n_cols + { + break; } path.push(field.name.clone()); @@ -591,8 +588,8 @@ mod tests { use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; use super::{ - lookup_value_stat, parse_stats_json_array, ColumnCountStat, ColumnValueStat, StatValue, - Stats, + ColumnCountStat, ColumnValueStat, StatValue, Stats, lookup_value_stat, + parse_stats_json_array, }; use crate::spec::fields::{ STATS_FIELD_MAX_VALUES, STATS_FIELD_MIN_VALUES, STATS_FIELD_NULL_COUNT, diff --git a/crates/sail-delta-lake/src/spec/utils.rs b/crates/sail-delta-lake/src/spec/utils.rs index c0b75ca80..4b899983e 100644 --- a/crates/sail-delta-lake/src/spec/utils.rs +++ b/crates/sail-delta-lake/src/spec/utils.rs @@ -20,7 +20,7 @@ use std::str::Utf8Error; use arrow_schema::extension::ExtensionType; use parquet_variant_compute::VariantType; -use percent_encoding::{percent_decode_str, percent_encode, AsciiSet, CONTROLS}; +use percent_encoding::{AsciiSet, CONTROLS, percent_decode_str, percent_encode}; use sail_common_datafusion::variant::is_marked_variant_storage_type; use super::schema::{DataType, StructField}; diff --git a/crates/sail-delta-lake/src/table/mod.rs b/crates/sail-delta-lake/src/table/mod.rs index 4a98b8bf8..45e83710f 100644 --- a/crates/sail-delta-lake/src/table/mod.rs +++ b/crates/sail-delta-lake/src/table/mod.rs @@ -28,44 +28,45 @@ use datafusion::catalog::Session; use datafusion::datasource::listing::ListingTableUrl; use datafusion_common::{DataFusionError, Result}; use futures::TryStreamExt; -use object_store::path::Path; use object_store::ObjectStore; +use object_store::path::Path; use sail_catalog::manager::CatalogManager; use sail_common_datafusion::catalog::delta::{ DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, }; -use sail_common_datafusion::catalog::{LakehouseExecutionContext, TableColumnStatus}; +use sail_common_datafusion::catalog::{ + CommitAuthority, LakehouseExecutionContext, TableColumnStatus, +}; use sail_common_datafusion::extension::SessionExtensionAccessor; use url::Url; use crate::catalog::coordinator::DeltaCatalogCommitCoordinator; use crate::catalog_managed::{ - catalog_managed_delta_table, metadata_with_catalog_managed, protocol_with_catalog_managed, - CatalogManagedDeltaTable, + CatalogManagedDeltaTable, catalog_managed_delta_table, metadata_with_catalog_managed, + protocol_with_catalog_managed, }; -use crate::datasource::{df_logical_schema, DeltaScanConfig}; +use crate::datasource::{DeltaScanConfig, df_logical_schema}; pub mod features; pub use features::{ ChangeDataFeedSupport, ChangeDataFeedToken, ColumnMappingToken, DeletionVectorToken, EnabledRowTrackingToken, RowTrackingToken, SupportedRowTrackingToken, }; -use crate::delta_log::{default_logstore, resolve_version_timestamp, LogStoreRef, StorageConfig}; +use crate::delta_log::{LogStoreRef, StorageConfig, default_logstore, resolve_version_timestamp}; use crate::logical::table_source::DeltaTableSource; -use crate::options::gen::DeltaReadOptions; +use crate::options::r#gen::DeltaReadOptions; use crate::schema::{ metadata_for_create_with_struct_type, normalize_delta_schema, protocol_for_create, schema_has_column_defaults, schema_has_generated_columns, schema_has_identity_columns, }; pub use crate::snapshot::DeltaSnapshot; use crate::snapshot::{ - catalog_managed_commit_file_name, CatalogManagedCommitFile, CatalogManagedCommitSet, - DeltaSnapshotConfig, + CatalogManagedCommitFile, CatalogManagedCommitSet, DeltaSnapshotConfig, + catalog_managed_commit_file_name, }; use crate::spec::{ - contains_timestampntz_arrow, contains_variant_arrow, CommitAction, DeltaError, - DeltaError as DeltaTableError, DeltaOperation, DeltaResult, Protocol, SaveMode, StructType, - TableFeature, + CommitAction, DeltaError, DeltaError as DeltaTableError, DeltaOperation, DeltaResult, Protocol, + SaveMode, StructType, TableFeature, contains_timestampntz_arrow, contains_variant_arrow, }; use crate::transaction::CommitBuilder; @@ -105,10 +106,10 @@ impl DeltaTable { /// Loads the DeltaTable state for the given version. pub async fn load_version(&mut self, version: i64) -> Result<(), DeltaTableError> { - if let Some(snapshot) = &self.state { - if snapshot.version() > version { - self.state = None; - } + if let Some(snapshot) = &self.state + && snapshot.version() > version + { + self.state = None; } self.update_incremental(Some(version)).await } @@ -388,6 +389,12 @@ fn is_missing_delta_log_error(error: &DeltaTableError) -> bool { ) } +pub(crate) fn catalog_managed_commit_context( + lakehouse_table: Option<&LakehouseExecutionContext>, +) -> Option<&LakehouseExecutionContext> { + lakehouse_table.filter(|context| context.commit == CommitAuthority::DeltaRatifiedCommit) +} + async fn load_catalog_managed_delta_bootstrap_info( ctx: &C, catalog_table: &[String], @@ -542,14 +549,15 @@ where .await?; let latest_table_version = response.latest_table_version; - if let Some(version) = version_as_of { - if latest_table_version >= 0 && version > latest_table_version { - return Err(DataFusionError::External(Box::new(DeltaTableError::generic( - format!( - "Catalog-managed Delta table latest ratified version is {latest_table_version}, but version {version} was requested" - ), - )))); - } + if let Some(version) = version_as_of + && latest_table_version >= 0 + && version > latest_table_version + { + return Err(DataFusionError::External(Box::new( + DeltaTableError::generic(format!( + "Catalog-managed Delta table latest ratified version is {latest_table_version}, but version {version} was requested" + )), + ))); } let Some(effective_end_version) = @@ -605,9 +613,11 @@ where if let Some(file_name) = staged_catalog_managed_commit_file(&log_store, version).await? { by_version.insert(version, CatalogManagedCommitFile { version, file_name }); } else { - return Err(DataFusionError::External(Box::new(DeltaTableError::generic( - format!("Catalog-managed Delta commit version {version} was not returned by the catalog"), - )))); + return Err(DataFusionError::External(Box::new( + DeltaTableError::generic(format!( + "Catalog-managed Delta commit version {version} was not returned by the catalog" + )), + ))); } } @@ -752,7 +762,7 @@ async fn load_delta_read_state( let log_store = create_logstore_with_object_store(object_store, table_url.clone(), storage_config)?; - let catalog_managed_commits = match lakehouse_table.as_ref() { + let catalog_managed_commits = match catalog_managed_commit_context(lakehouse_table.as_ref()) { Some(lakehouse_table) => { load_catalog_managed_commits_for_snapshot( &ctx, diff --git a/crates/sail-delta-lake/src/table_format.rs b/crates/sail-delta-lake/src/table_format.rs index 0717bb964..936f26acb 100644 --- a/crates/sail-delta-lake/src/table_format.rs +++ b/crates/sail-delta-lake/src/table_format.rs @@ -5,17 +5,17 @@ use async_trait::async_trait; use chrono::Utc; use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema, SchemaRef}; use datafusion::catalog::Session; -use datafusion::common::{not_impl_err, plan_err, DFSchema, DataFusionError, Result}; +use datafusion::common::{DFSchema, DataFusionError, Result, not_impl_err, plan_err}; use datafusion::datasource::listing::ListingTableUrl; -use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::execution::SessionState; +use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::logical_expr::{LogicalPlan, TableSource}; use datafusion::physical_plan::ExecutionPlan; use datafusion_expr::expr::Sort; use datafusion_expr::{Expr, Extension, UserDefinedLogicalNodeCore}; use educe::Educe; use sail_common_datafusion::catalog::delta::{ - unity_table_id_value, DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, + DELTA_UNITY_TABLE_ID_KEY, DELTA_UNITY_TABLE_ID_LEGACY_KEY, unity_table_id_value, }; use sail_common_datafusion::catalog::{ CatalogPartitionField, CatalogTableColumnIdentity, CommitAuthority, LakehouseExecutionContext, @@ -25,10 +25,10 @@ use sail_common_datafusion::column_features::{ ColumnFeatureKey, ColumnFeatures, SAIL_WRITE_TARGET_NULLABLE_METADATA_KEY, }; use sail_common_datafusion::datasource::{ - create_sort_order, find_path_in_options, BucketBy, DeleteInfo, MergeInfo, OptionLayer, - PhysicalSinkMode, SinkInfo, SinkMode, SourceInfo, TableFormat, TableFormatAlterTableOperation, + BucketBy, CATALOG_TABLE_OPTION, DeleteInfo, MergeInfo, OptionLayer, PhysicalSinkMode, SinkInfo, + SinkMode, SourceInfo, TableFormat, TableFormatAlterTableOperation, TableFormatCreateTableColumn, TableFormatCreateTableInfo, TableFormatCreateTableResult, - TableFormatMetadata, TableFormatRegistry, CATALOG_TABLE_OPTION, + TableFormatMetadata, TableFormatRegistry, create_sort_order, find_path_in_options, }; use sail_common_datafusion::streaming::event::schema::is_flow_event_schema; use sail_common_datafusion::utils::items::ItemTaker; @@ -41,7 +41,7 @@ use url::Url; use crate::catalog_managed::{metadata_with_catalog_managed, protocol_with_catalog_managed}; use crate::datasource::actions::adds_to_remove_actions; use crate::delta_log::StorageConfig; -use crate::options::gen::{DeltaReadOptions, DeltaWriteOptions}; +use crate::options::r#gen::{DeltaReadOptions, DeltaWriteOptions}; use crate::physical_plan::planner::{DeltaPhysicalPlanner, DeltaPlannerConfig, PlannerContext}; use crate::schema::type_widening::alter_column_type as alter_delta_column_type; use crate::schema::{ @@ -54,19 +54,18 @@ use crate::schema::{ }; use crate::snapshot::DeltaSnapshotConfig; use crate::spec::{ + ColumnMappingMode, ColumnMetadataKey, CommitAction, DataType as DeltaDataType, DeltaOperation, + MetadataValue, Protocol, SaveMode, StructField, StructType, TableFeature, TableProperties, canonicalize_and_validate_table_properties, contains_timestampntz_arrow, - contains_variant_arrow, route_table_property_key, ColumnMappingMode, ColumnMetadataKey, - CommitAction, DataType as DeltaDataType, DeltaOperation, MetadataValue, Protocol, SaveMode, - StructField, StructType, TableFeature, TableProperties, + contains_variant_arrow, route_table_property_key, }; use crate::table::{ - create_delta_table_with_object_store, create_logstore_with_object_store, - infer_delta_logical_metadata, infer_delta_logical_schema, + DeltaTable, catalog_managed_commit_context, create_delta_table_with_object_store, + create_logstore_with_object_store, infer_delta_logical_metadata, infer_delta_logical_schema, load_catalog_managed_commits_for_snapshot, open_table_with_object_store_and_table_config, - DeltaTable, }; use crate::transaction::CommitBuilder; -use crate::{create_delta_source, DeltaTableError}; +use crate::{DeltaTableError, create_delta_source}; /// Delta Lake implementation of [`TableFormat`]. #[derive(Debug)] @@ -647,13 +646,14 @@ pub(crate) async fn plan_delta_write( .object_store_registry .get_store(&table_url) .map_err(|e| DataFusionError::External(Box::new(e)))?; - let lakehouse_planning_context = lakehouse_table.as_ref().filter(|context| { - context.table_identity.table_id.is_some() - && !matches!( - context.operation, - LakehouseOperation::Create | LakehouseOperation::Register - ) - }); + let lakehouse_planning_context = catalog_managed_commit_context(lakehouse_table.as_ref()) + .filter(|context| { + context.table_identity.table_id.is_some() + && !matches!( + context.operation, + LakehouseOperation::Create | LakehouseOperation::Register + ) + }); let table = match open_delta_write_planning_table( ctx, table_url.clone(), @@ -701,35 +701,36 @@ pub(crate) async fn plan_delta_write( .as_ref() .map(|snapshot| snapshot.metadata().partition_columns().clone()); - if let Some(existing_partitions) = &existing_partition_columns { - if !partition_by.is_empty() && partition_by != *existing_partitions { - match mode { - PhysicalSinkMode::Append => { - return plan_err!( - "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \ + if let Some(existing_partitions) = &existing_partition_columns + && !partition_by.is_empty() + && partition_by != *existing_partitions + { + match mode { + PhysicalSinkMode::Append => { + return plan_err!( + "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \ Cannot change partitioning on append.", - existing_partitions, - partition_by - ); - } - PhysicalSinkMode::OverwriteIf { .. } => { - return plan_err!( - "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \ + existing_partitions, + partition_by + ); + } + PhysicalSinkMode::OverwriteIf { .. } => { + return plan_err!( + "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \ Cannot change partitioning with replaceWhere or conditional overwrite.", - existing_partitions, - partition_by - ); - } - PhysicalSinkMode::Overwrite if !delta_options.overwrite_schema => { - return plan_err!( - "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \ + existing_partitions, + partition_by + ); + } + PhysicalSinkMode::Overwrite if !delta_options.overwrite_schema => { + return plan_err!( + "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \ Set overwriteSchema=true to change partitioning.", - existing_partitions, - partition_by - ); - } - _ => {} + existing_partitions, + partition_by + ); } + _ => {} } } @@ -782,7 +783,7 @@ async fn open_delta_write_planning_table( ..Default::default() }; - if let Some(lakehouse_table) = lakehouse_table { + if let Some(lakehouse_table) = catalog_managed_commit_context(lakehouse_table) { let log_store = create_logstore_with_object_store( Arc::clone(&object_store), table_url.clone(), diff --git a/crates/sail-delta-lake/src/transaction/conflict_checker.rs b/crates/sail-delta-lake/src/transaction/conflict_checker.rs index 6ce51b7b4..509a74daf 100644 --- a/crates/sail-delta-lake/src/transaction/conflict_checker.rs +++ b/crates/sail-delta-lake/src/transaction/conflict_checker.rs @@ -22,11 +22,11 @@ use std::collections::HashSet; -use crate::delta_log::{get_actions, LogStore}; +use crate::delta_log::{LogStore, get_actions}; use crate::spec::{ - logical_file_key, Action, Add, CommitAction, CommitConflictError, CommitInfo, DeltaError, - DeltaOperation, DeltaResult, IsolationLevel, LogicalFileKey, Metadata, Protocol, Remove, - Transaction, + Action, Add, CommitAction, CommitConflictError, CommitInfo, DeltaError, DeltaOperation, + DeltaResult, IsolationLevel, LogicalFileKey, Metadata, Protocol, Remove, Transaction, + logical_file_key, }; use crate::table::DeltaSnapshot; @@ -342,9 +342,9 @@ impl<'a> ConflictChecker<'a> { self.txn_info.read_snapshot.protocol().min_writer_version(), ); if curr_read < win_read || win_write < curr_write { - return Err(CommitConflictError::ProtocolChanged( - format!("required read/write {win_read}/{win_write}, current read/write {curr_read}/{curr_write}"), - )); + return Err(CommitConflictError::ProtocolChanged(format!( + "required read/write {win_read}/{win_write}, current read/write {curr_read}/{curr_write}" + ))); }; } if !self.winning_commit_summary.protocol().is_empty() { diff --git a/crates/sail-delta-lake/src/transaction/mod.rs b/crates/sail-delta-lake/src/transaction/mod.rs index 23188718c..b14b7841b 100644 --- a/crates/sail-delta-lake/src/transaction/mod.rs +++ b/crates/sail-delta-lake/src/transaction/mod.rs @@ -36,13 +36,13 @@ use crate::checkpoint::{ }; use crate::delta_log::cleanup::cleanup_expired_delta_log_files; use crate::delta_log::{ - resolve_effective_protocol_and_metadata, resolve_version_timestamp, CommitOrBytes, LogStoreRef, - ObjectStoreRef, + CommitOrBytes, LogStoreRef, ObjectStoreRef, resolve_effective_protocol_and_metadata, + resolve_version_timestamp, }; use crate::snapshot::DeltaSnapshotConfig; use crate::spec::{ - checksum_path, staged_commit_path, temp_commit_path, Action, CommitAction, DeltaError, - DeltaOperation, DeltaResult, Metadata, TableFeature, Transaction, VersionChecksum, + Action, CommitAction, DeltaError, DeltaOperation, DeltaResult, Metadata, TableFeature, + Transaction, VersionChecksum, checksum_path, staged_commit_path, temp_commit_path, }; pub use crate::spec::{CommitConflictError, TransactionError}; use crate::table::DeltaSnapshot; @@ -138,7 +138,7 @@ impl OperationMetrics { pub fn into_map(self) -> HashMap { let mut out = self.extra; macro_rules! insert_opt { - ($key:literal, $field:expr) => { + ($key:literal, $field:expr_2021) => { if let Some(v) = $field { out.insert($key.to_string(), Value::from(v)); } @@ -347,12 +347,11 @@ impl OperationMetrics { if self.num_copied_rows.is_none() { self.num_copied_rows = self.num_output_rows; } - if self.num_deleted_rows.is_none() { - if let (Some(touched), Some(copied)) = + if self.num_deleted_rows.is_none() + && let (Some(touched), Some(copied)) = (self.num_touched_rows, self.num_copied_rows) - { - self.num_deleted_rows = Some(touched.saturating_sub(copied)); - } + { + self.num_deleted_rows = Some(touched.saturating_sub(copied)); } if self.rewrite_time_ms.is_none() { self.rewrite_time_ms = self.write_time_ms; @@ -380,15 +379,15 @@ impl OperationMetrics { if self.rewrite_time_ms.is_none() { self.rewrite_time_ms = self.write_time_ms; } - if self.num_target_rows_deleted.is_none() { - if let (Some(touched), Some(copied), Some(updated)) = ( + if self.num_target_rows_deleted.is_none() + && let (Some(touched), Some(copied), Some(updated)) = ( self.num_touched_rows, self.num_target_rows_copied, self.num_target_rows_updated, - ) { - self.num_target_rows_deleted = - Some(touched.saturating_sub(copied.saturating_add(updated))); - } + ) + { + self.num_target_rows_deleted = + Some(touched.saturating_sub(copied.saturating_add(updated))); } self.num_target_rows_inserted.get_or_insert(0); self.num_target_rows_updated.get_or_insert(0); @@ -776,27 +775,24 @@ fn finalize_attempt_actions( if read_snapshot.is_some() && !old_in_commit_timestamps_enabled && new_in_commit_timestamps_enabled + && let Some(in_commit_timestamp) = in_commit_timestamp + && let Some(metadata) = finalized_actions + .iter_mut() + .find_map(|action| match action { + CommitAction::Metadata(metadata) => Some(metadata), + _ => None, + }) { - if let Some(in_commit_timestamp) = in_commit_timestamp { - if let Some(metadata) = finalized_actions - .iter_mut() - .find_map(|action| match action { - CommitAction::Metadata(metadata) => Some(metadata), - _ => None, - }) - { - *metadata = metadata - .clone() - .add_config_key( - "delta.inCommitTimestampEnablementVersion".to_string(), - version.to_string(), - ) - .add_config_key( - "delta.inCommitTimestampEnablementTimestamp".to_string(), - in_commit_timestamp.to_string(), - ); - } - } + *metadata = metadata + .clone() + .add_config_key( + "delta.inCommitTimestampEnablementVersion".to_string(), + version.to_string(), + ) + .add_config_key( + "delta.inCommitTimestampEnablementTimestamp".to_string(), + in_commit_timestamp.to_string(), + ); } Ok(finalized_actions) @@ -880,13 +876,13 @@ fn validate_effective_commit_target( return Err(TransactionError::TableFeaturesRequired(TableFeature::DeletionVectors).into()); } - // TODO(cdf-writes): Data-changing operations still do not emit AddCDCFile actions. Commit-time - // protocol checks currently reject changeDataFeed tables before these writes can land, but once - // the feature is enabled here we need an operation-aware validation instead of relying on that. - if table_property_enabled(&metadata, "delta.enableChangeDataFeed") - && !protocol_supports_legacy_change_data_feed(&protocol) - && !protocol_has_writer_feature(&protocol, &TableFeature::ChangeDataFeed) - { + // TODO(cdf-writes): Data-changing operations still do not emit AddCDCFile actions. Until CDF + // write support is implemented, reject all writes to tables with CDF enabled, regardless of + // whether the protocol support is legacy (writer v4-6) or explicit (writer v7+ feature). + // Previously this guard only rejected v7+ tables, relying on `can_write_to_protocol` to reject + // legacy tables via implied-feature expansion. Now that legacy versions no longer expand + // implied features (matching delta-spark), this guard must cover legacy tables too. + if table_property_enabled(&metadata, "delta.enableChangeDataFeed") { return Err(TransactionError::TableFeaturesRequired(TableFeature::ChangeDataFeed).into()); } @@ -1288,15 +1284,13 @@ impl std::future::IntoFuture for PreparedCommit { // For creation attempts where the table now exists, ensure protocol/metadata match // and strip creation-only actions before retrying as an append. if creation_intent { - if let Some(txn_protocol) = creation_protocol.as_ref() { - if txn_protocol != snapshot.protocol() { - return Err(TransactionError::CommitConflict( - CommitConflictError::ProtocolChanged( - "protocol changed".into(), - ), - ) - .into()); - } + if let Some(txn_protocol) = creation_protocol.as_ref() + && txn_protocol != snapshot.protocol() + { + return Err(TransactionError::CommitConflict( + CommitConflictError::ProtocolChanged("protocol changed".into()), + ) + .into()); } let metadata_compatible = @@ -2037,9 +2031,9 @@ impl std::future::IntoFuture for PostCommit { Ok(s) => Some(Arc::new(s)), Err(e) => { warn!( - "Post-commit: failed to load state for version {} (post-commit activities skipped): {e}", - this.version - ); + "Post-commit: failed to load state for version {} (post-commit activities skipped): {e}", + this.version + ); None } }; @@ -2072,20 +2066,19 @@ impl std::future::IntoFuture for PostCommit { // before hook — best-effort: the commit entry has already been durably written // to the log, so a hook failure here does not roll back the transaction. - if let Some(handler) = &this.custom_execute_handler { - if let Err(e) = handler + if let Some(handler) = &this.custom_execute_handler + && let Err(e) = handler .before_post_commit_hook( &this.log_store, will_create_checkpoint, post_commit_operation_id, ) .await - { - warn!( - "before_post_commit_hook failed for version {}: {e}", - this.version - ); - } + { + warn!( + "before_post_commit_hook failed for version {}: {e}", + this.version + ); } // Checkpoint — best-effort: checkpoint creation is a performance optimization @@ -2113,29 +2106,30 @@ impl std::future::IntoFuture for PostCommit { // Log cleanup let mut num_log_files_cleaned_up: u64 = 0; - if cleanup_logs_setting && checkpoint_created { - if let Some(s) = state.as_ref() { - let retention_millis = - i64::try_from(s.table_properties().log_retention_duration().as_millis()) - .unwrap_or(i64::MAX); - let cutoff_timestamp = (Utc::now().timestamp_millis() - retention_millis) - .div_euclid(24 * 60 * 60 * 1000) - * (24 * 60 * 60 * 1000); - match cleanup_expired_delta_log_files( - s.as_ref(), - this.log_store.as_ref(), - cutoff_timestamp, - Some(post_commit_operation_id), - ) - .await - { - Ok(n) => num_log_files_cleaned_up = n as u64, - Err(e) => { - warn!( - "Failed to clean up expired log files for version {}: {e}", - this.version - ); - } + if cleanup_logs_setting + && checkpoint_created + && let Some(s) = state.as_ref() + { + let retention_millis = + i64::try_from(s.table_properties().log_retention_duration().as_millis()) + .unwrap_or(i64::MAX); + let cutoff_timestamp = (Utc::now().timestamp_millis() - retention_millis) + .div_euclid(24 * 60 * 60 * 1000) + * (24 * 60 * 60 * 1000); + match cleanup_expired_delta_log_files( + s.as_ref(), + this.log_store.as_ref(), + cutoff_timestamp, + Some(post_commit_operation_id), + ) + .await + { + Ok(n) => num_log_files_cleaned_up = n as u64, + Err(e) => { + warn!( + "Failed to clean up expired log files for version {}: {e}", + this.version + ); } } } @@ -2171,20 +2165,19 @@ impl std::future::IntoFuture for PostCommit { } // after hook - if let Some(handler) = &this.custom_execute_handler { - if let Err(e) = handler + if let Some(handler) = &this.custom_execute_handler + && let Err(e) = handler .after_post_commit_hook( &this.log_store, checkpoint_created, post_commit_operation_id, ) .await - { - warn!( - "after_post_commit_hook failed for version {}: {e}", - this.version - ); - } + { + warn!( + "after_post_commit_hook failed for version {}: {e}", + this.version + ); } Ok(FinalizedCommit { @@ -2205,16 +2198,16 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use object_store::memory::InMemory; use object_store::ObjectStore; + use object_store::memory::InMemory; use url::Url; use super::*; - use crate::delta_log::{default_logstore, get_actions, StorageConfig}; + use crate::delta_log::{StorageConfig, default_logstore, get_actions}; use crate::schema::protocol_for_create; use crate::spec::{ - checksum_path, Action, CommitAction, CommitInfo, DataType, DeltaError, DomainMetadata, - Metadata, Protocol, SaveMode, StructField, StructType, TableFeature, VersionChecksum, + Action, CommitAction, CommitInfo, DataType, DeltaError, DomainMetadata, Metadata, Protocol, + SaveMode, StructField, StructType, TableFeature, VersionChecksum, checksum_path, }; fn test_log_store(store: Arc) -> LogStoreRef { diff --git a/crates/sail-delta-lake/src/transaction/protocol.rs b/crates/sail-delta-lake/src/transaction/protocol.rs index bfc545daf..0a46b6dfd 100644 --- a/crates/sail-delta-lake/src/transaction/protocol.rs +++ b/crates/sail-delta-lake/src/transaction/protocol.rs @@ -22,53 +22,11 @@ use std::collections::HashSet; use std::sync::LazyLock; use crate::spec::{ - contains_timestampntz, contains_variant, Action, CommitConflictError, DeltaOperation, Protocol, - Schema, TableFeature, TransactionError, + Action, CommitConflictError, DeltaOperation, Protocol, Schema, TableFeature, TransactionError, + contains_timestampntz, contains_variant, }; use crate::table::DeltaSnapshot; -static READER_V2: LazyLock> = - LazyLock::new(|| HashSet::from_iter([TableFeature::ColumnMapping])); -static WRITER_V2: LazyLock> = - LazyLock::new(|| HashSet::from_iter([TableFeature::AppendOnly, TableFeature::Invariants])); -static WRITER_V3: LazyLock> = LazyLock::new(|| { - HashSet::from_iter([ - TableFeature::AppendOnly, - TableFeature::Invariants, - TableFeature::CheckConstraints, - ]) -}); -static WRITER_V4: LazyLock> = LazyLock::new(|| { - HashSet::from_iter([ - TableFeature::AppendOnly, - TableFeature::Invariants, - TableFeature::CheckConstraints, - TableFeature::ChangeDataFeed, - TableFeature::GeneratedColumns, - ]) -}); -static WRITER_V5: LazyLock> = LazyLock::new(|| { - HashSet::from_iter([ - TableFeature::AppendOnly, - TableFeature::Invariants, - TableFeature::CheckConstraints, - TableFeature::ChangeDataFeed, - TableFeature::GeneratedColumns, - TableFeature::ColumnMapping, - ]) -}); -static WRITER_V6: LazyLock> = LazyLock::new(|| { - HashSet::from_iter([ - TableFeature::AppendOnly, - TableFeature::Invariants, - TableFeature::CheckConstraints, - TableFeature::ChangeDataFeed, - TableFeature::GeneratedColumns, - TableFeature::ColumnMapping, - TableFeature::IdentityColumns, - ]) -}); - pub struct ProtocolChecker { reader_features: HashSet, writer_features: HashSet, @@ -107,9 +65,13 @@ impl ProtocolChecker { &self, protocol: &Protocol, ) -> Result>, TransactionError> { + // Legacy reader versions (1, 2) do not carry explicit feature lists. The version + // number implies a feature set (e.g. reader v2 implies columnMapping), but — + // matching delta-spark's `protocolCheck` — we do not expand the implied set here. + // Whether a legacy-implied feature is actually *active* is validated separately + // at operation time (e.g. `verify_column_mapping`). match protocol.min_reader_version() { - 0 | 1 => Ok(None), - 2 => Ok(Some(READER_V2.clone())), + 0..=2 => Ok(None), 3 => Ok(Some( protocol .reader_features() @@ -128,16 +90,19 @@ impl ProtocolChecker { &self, protocol: &Protocol, ) -> Result>, TransactionError> { - // Delta protocol differs here: - // - writer versions 2..=6 imply a fixed feature set from `minWriterVersion` - // - writer version 7 uses the explicit `writerFeatures` declared by the table + // Legacy writer versions (1..=6) do not carry explicit feature lists. The version + // number implies a feature set (e.g. writer v5 implies columnMapping, changeDataFeed, + // ...), but — matching delta-spark's `protocolCheck` — we do not expand the implied + // set here. Whether a legacy-implied feature is actually *active* is validated + // separately at operation time (e.g. append-only check, CDF guard). + // + // Previously this method expanded the full implied set for each version and required + // the writer to support every implied feature. That was stricter than delta-spark + // and blocked writes to tables that did not actually use the unsupported features + // (e.g. a v5 column-mapping table without CDF enabled was rejected because CDF is + // implied by v5 but not implemented). See lakehq/sail#2041. match protocol.min_writer_version() { - 0 | 1 => Ok(None), - 2 => Ok(Some(WRITER_V2.clone())), - 3 => Ok(Some(WRITER_V3.clone())), - 4 => Ok(Some(WRITER_V4.clone())), - 5 => Ok(Some(WRITER_V5.clone())), - 6 => Ok(Some(WRITER_V6.clone())), + 0..=6 => Ok(None), 7 => Ok(Some( protocol .writer_features() @@ -346,8 +311,9 @@ pub static INSTANCE: LazyLock = LazyLock::new(|| { reader_features.insert(TableFeature::CatalogManaged); let mut writer_features = HashSet::new(); // Keep this list aligned with end-to-end behavior, not just protocol parsing. - // For writer versions 2..=6, claiming support here also means accepting older tables whose - // `minWriterVersion` implies the feature set in WRITER_V2..WRITER_V6. + // These features are only checked against the explicit `writerFeatures` list on + // writer version 7+ tables. Legacy tables (writer version 1..=6) do not have + // explicit feature lists; their implied features are validated at operation time. writer_features.insert(TableFeature::AppendOnly); writer_features.insert(TableFeature::InCommitTimestamp); writer_features.insert(TableFeature::TimestampWithoutTimezone); @@ -481,4 +447,111 @@ mod tests { TransactionError::TableFeaturesRequired(TableFeature::VariantShredding) )); } + + // --- Legacy protocol version tests (writer versions 1..=6) --- + // + // These tests verify that the protocol checker does NOT expand implied feature + // sets for legacy writer versions. A table at writer version 5 implies support + // for ChangeDataFeed, but if CDF is not enabled the write should succeed even + // though sail does not implement CDF writes. See lakehq/sail#2041. + + #[test] + fn global_checker_accepts_legacy_writer_v5_without_explicit_features() { + // Simulates a delta-spark-created table with column mapping mode=name. + // delta-spark produces Protocol(2, 5) with no explicit feature lists. + let protocol = Protocol::new(2, 5, None, None); + + INSTANCE.can_read_from_protocol(&protocol).unwrap(); + INSTANCE.can_write_to_protocol(&protocol).unwrap(); + } + + #[test] + fn global_checker_accepts_legacy_writer_v4_without_explicit_features() { + // Writer v4 implies ChangeDataFeed + GeneratedColumns, but we should not + // require them unless they are actually active on the table. + let protocol = Protocol::new(1, 4, None, None); + + INSTANCE.can_read_from_protocol(&protocol).unwrap(); + INSTANCE.can_write_to_protocol(&protocol).unwrap(); + } + + #[test] + fn global_checker_accepts_legacy_writer_v6_without_explicit_features() { + // Writer v6 implies IdentityColumns on top of v5 features. + let protocol = Protocol::new(2, 6, None, None); + + INSTANCE.can_read_from_protocol(&protocol).unwrap(); + INSTANCE.can_write_to_protocol(&protocol).unwrap(); + } + + #[test] + fn global_checker_accepts_legacy_writer_v2_and_v3() { + let v2 = Protocol::new(1, 2, None, None); + INSTANCE.can_write_to_protocol(&v2).unwrap(); + + let v3 = Protocol::new(1, 3, None, None); + INSTANCE.can_write_to_protocol(&v3).unwrap(); + } + + #[test] + fn global_checker_accepts_legacy_reader_v2_without_explicit_features() { + // Reader v2 implies ColumnMapping, but we should not require it at the + // protocol-check level. Column mapping activation is verified separately. + let protocol = Protocol::new(2, 2, None, None); + + INSTANCE.can_read_from_protocol(&protocol).unwrap(); + INSTANCE.can_write_to_protocol(&protocol).unwrap(); + } + + #[test] + fn global_checker_rejects_v7_with_unsupported_change_data_feed_feature() { + // Writer v7 with explicit changeDataFeed writer feature should still be + // rejected, since sail does not implement CDF writes. + let protocol = Protocol::new( + 3, + 7, + Some(vec![TableFeature::ColumnMapping]), + Some(vec![ + TableFeature::ColumnMapping, + TableFeature::ChangeDataFeed, + ]), + ); + + let err = INSTANCE.can_write_to_protocol(&protocol).unwrap_err(); + assert!(matches!( + err, + TransactionError::UnsupportedTableFeatures(features) + if features.contains(&TableFeature::ChangeDataFeed) + )); + } + + #[test] + fn global_checker_rejects_v7_with_unsupported_reader_feature() { + // Reader v3 with an unsupported explicit reader feature should be rejected. + let protocol = Protocol::new( + 3, + 7, + Some(vec![TableFeature::RowTracking]), + Some(vec![TableFeature::AppendOnly]), + ); + + let err = INSTANCE.can_read_from_protocol(&protocol).unwrap_err(); + assert!(matches!( + err, + TransactionError::UnsupportedTableFeatures(features) + if features.contains(&TableFeature::RowTracking) + )); + } + + #[test] + fn global_checker_rejects_unknown_writer_version() { + let protocol = Protocol::new(1, 99, None, None); + assert!(INSTANCE.can_write_to_protocol(&protocol).is_err()); + } + + #[test] + fn global_checker_rejects_unknown_reader_version() { + let protocol = Protocol::new(99, 7, None, Some(vec![])); + assert!(INSTANCE.can_read_from_protocol(&protocol).is_err()); + } } diff --git a/crates/sail-delta-lake/src/writer/mod.rs b/crates/sail-delta-lake/src/writer/mod.rs index f371e4471..d1d17e628 100644 --- a/crates/sail-delta-lake/src/writer/mod.rs +++ b/crates/sail-delta-lake/src/writer/mod.rs @@ -44,8 +44,8 @@ use partitioning::partition_ranges; use stats::create_add; use uuid::Uuid; use variant_shredding::{ - apply_variant_shredding_plan, build_variant_shredding_plan, VariantShreddingConfig, - VariantShreddingPlan, + VariantShreddingConfig, VariantShreddingPlan, apply_variant_shredding_plan, + build_variant_shredding_plan, }; use crate::conversion::ScalarExt; @@ -713,9 +713,9 @@ mod tests { use datafusion::arrow::array::{ArrayRef, Int32Array, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; + use object_store::ObjectStore; use object_store::memory::InMemory; use object_store::path::Path; - use object_store::ObjectStore; use parquet_variant::{Variant, VariantMetadata}; use parquet_variant_compute::json_to_variant; use parquet_variant_json::VariantToJson; @@ -828,8 +828,8 @@ mod tests { } #[tokio::test] - async fn variant_shredding_writer_drains_pending_batches_sequentially( - ) -> Result<(), DeltaTableError> { + async fn variant_shredding_writer_drains_pending_batches_sequentially() + -> Result<(), DeltaTableError> { let object_store: Arc = Arc::new(InMemory::new()); let table_path = Path::from("delta_table"); let batch1 = make_variant_batch(vec![Some(r#"{"a":1,"b":"delta","nested":{"c":7}}"#)])?; @@ -899,8 +899,8 @@ mod tests { } #[tokio::test] - async fn partition_path_with_percent_encoded_value_is_not_double_encoded( - ) -> Result<(), DeltaTableError> { + async fn partition_path_with_percent_encoded_value_is_not_double_encoded() + -> Result<(), DeltaTableError> { use datafusion::arrow::datatypes::TimeUnit; let object_store: Arc = Arc::new(InMemory::new()); diff --git a/crates/sail-delta-lake/src/writer/stats.rs b/crates/sail-delta-lake/src/writer/stats.rs index 867b578b6..cd71119d9 100644 --- a/crates/sail-delta-lake/src/writer/stats.rs +++ b/crates/sail-delta-lake/src/writer/stats.rs @@ -156,7 +156,7 @@ fn stats_from_metadata( if is_binary { warn!( "Skipping column {} because it's a binary field.", - &column_descr.name().to_string() + column_descr.name() ); None } else { @@ -364,8 +364,8 @@ fn apply_variant_stats_from_footer( } fn contains_variant_array_path(path: &[String]) -> bool { - path.windows(2) - .any(|parts| parts[0] == "list" && parts[1] == "element") + path.array_windows::<2>() + .any(|[left, right]| left == "list" && right == "element") } fn normalized_variant_stats_path(path: &[String]) -> Result { @@ -481,7 +481,7 @@ impl StatsScalar { use_min: bool, ) -> Result { macro_rules! get_stat { - ($val: expr) => { + ($val: expr_2021) => { if use_min { *$val.min_opt().unwrap() } else { @@ -740,7 +740,7 @@ fn get_list_field_name(column_descr: &Arc) -> Option { ("list", seen, _) if seen == max_rep_levels => return Some("list".to_string()), ("element", _, seen) if seen == max_rep_levels => return Some("element".to_string()), (SAIL_LIST_FIELD_NAME, _, seen) if seen == max_rep_levels => { - return Some(SAIL_LIST_FIELD_NAME.to_string()) + return Some(SAIL_LIST_FIELD_NAME.to_string()); } ("list", _, _) => lists_seen += 1, ("element", _, _) | (SAIL_LIST_FIELD_NAME, _, _) => items_seen += 1, diff --git a/crates/sail-delta-lake/src/writer/variant_shredding.rs b/crates/sail-delta-lake/src/writer/variant_shredding.rs index d216a202e..e5b83e37c 100644 --- a/crates/sail-delta-lake/src/writer/variant_shredding.rs +++ b/crates/sail-delta-lake/src/writer/variant_shredding.rs @@ -1,4 +1,4 @@ pub use sail_common_datafusion::variant::{ - apply_variant_shredding_plan, build_variant_shredding_plan, variant_top_level_columns, - VariantShreddingConfig, VariantShreddingPlan, + VariantShreddingConfig, VariantShreddingPlan, apply_variant_shredding_plan, + build_variant_shredding_plan, variant_top_level_columns, }; diff --git a/crates/sail-execution/Cargo.toml b/crates/sail-execution/Cargo.toml index ea3a17c4e..da260815d 100644 --- a/crates/sail-execution/Cargo.toml +++ b/crates/sail-execution/Cargo.toml @@ -11,7 +11,6 @@ sail-common = { path = "../sail-common" } sail-common-datafusion = { path = "../sail-common-datafusion" } sail-catalog = { path = "../sail-catalog" } sail-data-source = { path = "../sail-data-source" } -sail-server = { path = "../sail-server" } sail-python-udf = { path = "../sail-python-udf" } sail-delta-lake = { path = "../sail-delta-lake" } sail-iceberg = { path = "../sail-iceberg" } @@ -20,6 +19,7 @@ sail-logical-plan = { path = "../sail-logical-plan" } sail-physical-plan = { path = "../sail-physical-plan" } sail-telemetry = { path = "../sail-telemetry" } sail-catalog-system = { path = "../sail-catalog-system" } +sail-object-store = { path = "../sail-object-store" } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/sail-execution/proto/sail/driver/service.proto b/crates/sail-execution/proto/sail/driver/service.proto index f2afa7a29..a8ff28ed4 100644 --- a/crates/sail-execution/proto/sail/driver/service.proto +++ b/crates/sail-execution/proto/sail/driver/service.proto @@ -3,39 +3,43 @@ syntax = "proto3"; package sail.driver; message RegisterWorkerRequest { - uint64 worker_id = 1; - string host = 2; - uint32 port = 3; + uint64 driver_id = 1; + uint64 worker_id = 2; + string host = 3; + uint32 port = 4; } message RegisterWorkerResponse { } message ReportWorkerHeartbeatRequest { - uint64 worker_id = 1; + uint64 driver_id = 1; + uint64 worker_id = 2; } message ReportWorkerHeartbeatResponse { } message ReportWorkerKnownPeersRequest { - uint64 worker_id = 1; - repeated uint64 peer_worker_ids = 2; + uint64 driver_id = 1; + uint64 worker_id = 2; + repeated uint64 peer_worker_ids = 3; } message ReportWorkerKnownPeersResponse { } message ReportTaskStatusRequest { - uint64 job_id = 1; - uint64 stage = 2; - uint64 partition = 3; - uint64 attempt = 4; - TaskStatus status = 5; + uint64 driver_id = 1; + uint64 job_id = 2; + uint64 stage = 3; + uint64 partition = 4; + uint64 attempt = 5; + TaskStatus status = 6; // An optional explanation of the task status. - optional string message = 6; + optional string message = 7; // An optional JSON string that contains the cause of the task failure. - optional string cause = 7; + optional string cause = 8; uint64 sequence = 101; } diff --git a/crates/sail-execution/proto/sail/plan/physical.proto b/crates/sail-execution/proto/sail/plan/physical.proto index d8a3a2f60..331bdcd0b 100644 --- a/crates/sail-execution/proto/sail/plan/physical.proto +++ b/crates/sail-execution/proto/sail/plan/physical.proto @@ -58,15 +58,39 @@ message ExtendedPhysicalPlanNode { PythonDataSourceWriteCommitExecNode python_data_source_write_commit = 43; CatalogCommandExecNode catalog_command = 44; BarrierExecNode barrier = 45; - RelaxedTzCastExecNode relaxed_tz_cast = 46; - SparkPartitionIdExecNode spark_partition_id = 47; - DeletionVectorWriterExecNode deletion_vector_writer = 48; - DeletionVectorRowsWriterExecNode deletion_vector_rows_writer = 49; - CoalesceExecNode coalesce = 50; - FileDeleteExecNode file_delete = 51; + SparkPartitionIdExecNode spark_partition_id = 46; + DeletionVectorWriterExecNode deletion_vector_writer = 47; + DeletionVectorRowsWriterExecNode deletion_vector_rows_writer = 48; + CoalesceExecNode coalesce = 49; + FileDeleteExecNode file_delete = 50; + NoopSinkExecNode noop_sink = 51; + CsvExecNode csv = 52; + ParquetExecNode parquet = 53; + RemoteCheckpointWriteExecNode remote_checkpoint_write = 54; + RemoteCheckpointCommitExecNode remote_checkpoint_commit = 55; + IcebergMergeMetadataExecNode iceberg_merge_metadata = 56; + IcebergEqualityDeleteWriterExecNode iceberg_equality_delete_writer = 57; } } +message RemoteCheckpointWriteExecNode { + bytes input = 1; + string object_store_url = 2; + string prefix = 3; + bytes storage_schema = 4; +} + +message RemoteCheckpointCommitExecNode { + bytes input = 1; + string relation_id = 2; + string object_store_url = 3; + string prefix = 4; + bytes logical_schema = 5; + bytes storage_schema = 6; + bytes output_partitioning = 7; + LexOrdering output_ordering = 8; +} + message ExtendedPhysicalExprNode { oneof ExprKind { CastColumnExprNode schema_evolution_cast = 1; @@ -76,9 +100,22 @@ message ExtendedPhysicalExprNode { } } +enum StructFieldMatching { + STRUCT_FIELD_MATCHING_NAME = 0; + STRUCT_FIELD_MATCHING_PHYSICAL_NAME = 1; + STRUCT_FIELD_MATCHING_FIELD_ID = 2; +} + +enum SchemaEvolutionTimezoneMode { + SCHEMA_EVOLUTION_TIMEZONE_MODE_STRICT = 0; + SCHEMA_EVOLUTION_TIMEZONE_MODE_RELAXED = 1; +} + message CastColumnExprNode { bytes input_schema = 1; bytes target_schema = 2; + StructFieldMatching struct_field_matching = 3; + SchemaEvolutionTimezoneMode timezone_mode = 4; } message LambdaExprNode { @@ -178,6 +215,11 @@ message ExtendedScalarUdf { SparkToXmlUdf spark_to_xml = 29; SparkBinUdf spark_bin = 30; SparkNegativeUdf spark_negative = 31; + SparkFromXmlUdf spark_from_xml = 32; + SparkCeilUdf spark_ceil = 33; + SparkFloorUdf spark_floor = 34; + SparkTryToTimestampUdf spark_try_to_timestamp = 35; + SparkDateFormatUdf spark_date_format = 36; } } @@ -194,9 +236,14 @@ message ExtendedAggregateUdf { PySparkGroupAggUdaf py_spark_group_agg = 2; PySparkGroupMapUdaf py_spark_group_map = 3; PySparkBatchCollectorUdaf py_spark_batch_collector = 4; + PercentileDiscUdaf percentile_disc = 5; } } +message PercentileDiscUdaf { + bool ansi_mode = 1; +} + message StandardUdf {} message PySparkUdf { @@ -235,11 +282,20 @@ message XpathTypedUdf { } message SparkToXmlUdf { - string session_timezone = 1; + string session_timezone = 1; +} + +message SparkFromXmlUdf { + string session_timezone = 1; } message SparkUnixTimestampUdf { - string timezone = 1; + string session_timezone = 1; + bool ansi_mode = 2; +} + +message SparkDateFormatUdf { + string session_timezone = 1; } message StructFunctionUdf { @@ -255,23 +311,36 @@ message UpdateStructFieldUdf { } message TimestampNowUdf { - string timezone = 1; + string session_timezone = 1; string time_unit = 2; } message SparkTimestampUdf { optional string timezone = 1; bool is_try = 2; + bool ansi_mode = 3; } message SparkDateUdf { bool is_try = 1; } +message SparkTryToTimestampUdf { + optional string timezone = 1; +} + message SparkVariantGetUdf { bool safe = 1; } +message SparkCeilUdf { + bool ansi_mode = 1; +} + +message SparkFloorUdf { + bool ansi_mode = 1; +} + message SparkNextDayUdf { bool ansi_mode = 1; } @@ -345,11 +414,17 @@ message StandardUdaf {} message ExtendedWindowUdf { oneof UdwfKind { StandardUdwf standard = 1; + SparkFirstLastValueUdwf spark_first_last_value = 2; } } message StandardUdwf {} +message SparkFirstLastValueUdwf { + bool first = 1; + bool ignore_nulls = 2; +} + // Distinguishes Pandas (202) vs Arrow-native (252) grouped aggregate UDFs enum PySparkGroupAggKind { PY_SPARK_GROUP_AGG_KIND_PANDAS = 0; @@ -440,6 +515,8 @@ message PySparkUdfConfig { bool python_udtf_pandas_conversion_enabled = 7; bool python_udf_pandas_int_to_decimal_coercion_enabled = 8; bool binary_as_bytes = 9; + bool python_udf_pandas_prefer_int_extension_dtype = 10; + bool arrow_use_large_var_types = 11; } message EquivalenceProperties { @@ -557,6 +634,7 @@ message MemoryExecNode { bool show_sizes = 4; repeated LexOrdering sort_information = 5; optional uint64 limit = 6; + bytes output_partitioning = 7; } message ValuesExecNode { @@ -569,6 +647,20 @@ message NdJsonExecNode { CompressionTypeVariant file_compression_type = 2; } +message CsvExecNode { + bytes base_config = 1; + bytes options = 2; +} + +message ParquetExecNode { + bytes base_config = 1; + bytes options = 2; + optional bytes predicate = 3; + bytes output_partitioning = 4; + StructFieldMatching struct_field_matching = 5; + SchemaEvolutionTimezoneMode timezone_mode = 6; +} + message ArrowExecNode { bytes base_config = 1; } @@ -582,7 +674,6 @@ message TextExecNode { message BinarySourceExecNode { bytes base_config = 1; - optional string path_glob_filter = 2; } message AvroExecNode { @@ -599,6 +690,7 @@ message RecursiveQueryExecNode { bytes static_term = 2; bytes recursive_term = 3; bool is_distinct = 4; + bytes output_schema = 5; } message SortMergeJoinExecNode { @@ -704,6 +796,7 @@ message DeltaScanByAddsExecNode { int64 version = 9; optional bytes statistics = 10; string lakehouse_table_json = 11; + string catalog_managed_commits_json = 12; // TODO: Consider snapshot hints (partition columns, kernel schema) to avoid worker log reads. } @@ -731,6 +824,10 @@ message ConsoleSinkExecNode { bytes input = 1; } +message NoopSinkExecNode { + bytes input = 1; +} + message SocketSourceExecNode { string host = 1; uint32 port = 2; @@ -804,11 +901,6 @@ message CoalesceExecNode { uint64 output_partitions = 2; } -message RelaxedTzCastExecNode { - bytes input = 1; - bytes schema = 2; -} - message JoinOn { bytes left = 1; bytes right = 2; @@ -868,12 +960,25 @@ message IcebergWriterExecNode { string options = 6; bytes logical_input_schema = 7; string lakehouse_table_json = 8; + bool merge_row_intents = 9; +} + +enum IcebergSnapshotUpdateKind { + ICEBERG_SNAPSHOT_UPDATE_KIND_UNSPECIFIED = 0; + ICEBERG_SNAPSHOT_UPDATE_KIND_FAST_APPEND = 1; + ICEBERG_SNAPSHOT_UPDATE_KIND_FULL_OVERWRITE = 2; + ICEBERG_SNAPSHOT_UPDATE_KIND_ROW_DELTA = 3; } message IcebergCommitExecNode { bytes input = 1; string table_url = 2; string lakehouse_table_json = 3; + // Whether commit must validate the target snapshot captured while planning a row-level write. + bool validate_read_snapshot = 4; + // Absent when validation is required and the target had no current snapshot. + optional int64 expected_snapshot_id = 5; + IcebergSnapshotUpdateKind snapshot_update_kind = 6; } message IcebergManifestScanExecNode { @@ -904,6 +1009,24 @@ message IcebergDeleteApplyExecNode { string iceberg_schema_json = 6; } +message IcebergMergeMetadataExecNode { + bytes input = 1; + string data_file_path = 2; + optional string file_column_name = 3; + optional string row_index_column_name = 4; + optional int32 data_file_partition_spec_id = 5; + optional string data_file_partition_json = 6; +} + +message IcebergEqualityDeleteWriterExecNode { + bytes input = 1; + string table_url = 2; + string table_properties_json = 3; + optional string write_data_path = 4; + optional string write_folder_storage_path = 5; + string lakehouse_table_json = 6; +} + message DeltaLogReplayExecNode { // Legacy single-input encoding for sort replay mode. bytes input = 1; diff --git a/crates/sail-execution/proto/sail/stream/common.proto b/crates/sail-execution/proto/sail/stream/common.proto index 79ea8524d..cb36a11fa 100644 --- a/crates/sail-execution/proto/sail/stream/common.proto +++ b/crates/sail-execution/proto/sail/stream/common.proto @@ -9,3 +9,8 @@ message TaskStreamTicket { uint64 attempt = 4; uint64 channel = 5; } + +message DriverTaskStreamTicket { + uint64 driver_id = 1; + TaskStreamTicket stream = 2; +} diff --git a/crates/sail-execution/proto/sail/task/common.proto b/crates/sail-execution/proto/sail/task/common.proto index 6d15940cf..2adeaa946 100644 --- a/crates/sail-execution/proto/sail/task/common.proto +++ b/crates/sail-execution/proto/sail/task/common.proto @@ -52,7 +52,6 @@ message TaskInputWorkerKeyList { } message TaskInputRemoteLocator { - string uri = 1; uint64 stage = 2; repeated TaskInputRemoteKeyList keys = 3; } @@ -105,6 +104,4 @@ message TaskOutputLocalLocator { } message TaskOutputRemoteLocator { - string uri = 1; } - diff --git a/crates/sail-execution/src/driver/actor/core.rs b/crates/sail-execution/src/driver/actor/core.rs index 40d841e7e..393a7d17d 100644 --- a/crates/sail-execution/src/driver/actor/core.rs +++ b/crates/sail-execution/src/driver/actor/core.rs @@ -1,66 +1,50 @@ use std::collections::HashMap; -use std::mem; -use fastrace::future::FutureExt; -use fastrace::Span; use log::{error, info}; -use sail_server::actor::{Actor, ActorAction, ActorContext}; +use sail_common::actor::{Actor, ActorAction, ActorContext}; use crate::driver::job_scheduler::{JobScheduler, JobSchedulerOptions}; use crate::driver::task_assigner::{TaskAssigner, TaskAssignerOptions}; use crate::driver::worker_pool::{WorkerPool, WorkerPoolOptions}; -use crate::driver::{DriverActor, DriverEvent, DriverOptions}; -use crate::rpc::ServerMonitor; +use crate::driver::{DriverActor, DriverComponents, DriverEvent, DriverOptions}; use crate::stream_manager::{StreamManager, StreamManagerOptions}; use crate::task_runner::TaskRunner; #[tonic::async_trait] impl Actor for DriverActor { type Message = DriverEvent; - type Options = DriverOptions; + type Options = (DriverOptions, DriverComponents); fn name() -> &'static str { "DriverActor" } - fn new(options: DriverOptions) -> Self { - let worker_pool = WorkerPool::new( - options.worker_manager.clone(), - WorkerPoolOptions::from(&options), - ); + fn new(options: Self::Options) -> Self { + let (options, components) = options; + let DriverComponents { + worker_manager, + history_reporter, + } = components; + let worker_pool = WorkerPool::new(worker_manager, WorkerPoolOptions::from(&options)); let job_scheduler = JobScheduler::new(JobSchedulerOptions::from(&options)); let task_assigner = TaskAssigner::new(TaskAssignerOptions::from(&options)); let stream_manager = StreamManager::new(StreamManagerOptions::from(&options)); Self { options, - server: ServerMonitor::new(), + history_reporter, worker_pool, job_scheduler, task_assigner, task_runner: TaskRunner::new(), stream_manager, task_sequences: HashMap::new(), - history: None, + shutdown_notifier: None, } } - async fn start(&mut self, ctx: &mut ActorContext) { - let addr = ( - self.options.driver_listen_host.clone(), - self.options.driver_listen_port, - ); - let server = mem::take(&mut self.server); - let span = Span::enter_with_local_parent("DriverActor::serve"); - self.server = server - .start(Self::serve(ctx.handle().clone(), addr).in_span(span)) - .await; - } - fn receive(&mut self, ctx: &mut ActorContext, message: DriverEvent) -> ActorAction { match message { - DriverEvent::ServerReady { port, signal } => { - self.handle_server_ready(ctx, port, signal) - } + DriverEvent::Activate => self.handle_activate(ctx), DriverEvent::RegisterWorker { worker_id, host, @@ -107,11 +91,11 @@ impl Actor for DriverActor { result, } => self.handle_create_local_stream(ctx, key, storage, schema, result), DriverEvent::CreateRemoteStream { - uri, key, schema, + context, result, - } => self.handle_create_remote_stream(ctx, uri, key, schema, result), + } => self.handle_create_remote_stream(ctx, key, schema, context, result), DriverEvent::FetchDriverStream { key, result } => { self.handle_fetch_driver_stream(ctx, key, result) } @@ -122,13 +106,13 @@ impl Actor for DriverActor { result, } => self.handle_fetch_worker_stream(ctx, worker_id, key, schema, result), DriverEvent::FetchRemoteStream { - uri, key, schema, + context, result, - } => self.handle_fetch_remote_stream(ctx, uri, key, schema, result), + } => self.handle_fetch_remote_stream(ctx, key, schema, context, result), DriverEvent::ObserveState { observer } => self.handle_observe_state(ctx, observer), - DriverEvent::Shutdown { history } => self.handle_shutdown(ctx, history), + DriverEvent::Shutdown { result } => self.handle_shutdown(ctx, result), } } @@ -138,11 +122,12 @@ impl Actor for DriverActor { if let Err(e) = self.worker_pool.close(ctx).await { error!("encountered error while stopping workers: {e}"); } - if let Some(history) = self.history.take() { - let _ = history.send(self.build_history()); + ctx.children_mut().join().await; + let history = self.build_history(); + self.history_reporter.report(history).await; + if let Some(result) = self.shutdown_notifier.take() { + let _ = result.send(()); } - info!("stopping driver server"); - self.server.stop().await; - info!("driver server has stopped"); + info!("driver {} has stopped", self.options.driver_id); } } diff --git a/crates/sail-execution/src/driver/actor/handler.rs b/crates/sail-execution/src/driver/actor/handler.rs index e6e6b34fb..b83d3d6c3 100644 --- a/crates/sail-execution/src/driver/actor/handler.rs +++ b/crates/sail-execution/src/driver/actor/handler.rs @@ -1,18 +1,17 @@ use std::collections::HashSet; -use std::mem; use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::ExecutionPlan; use futures::TryStreamExt; -use log::{debug, error, info, warn}; +use log::{debug, info, warn}; +use sail_common::actor::{ActorAction, ActorContext}; use sail_common_datafusion::error::CommonErrorCause; use sail_common_datafusion::session::job::JobRunnerHistory; use sail_common_datafusion::system::observable::JobRunnerObserver; use sail_common_datafusion::system::predicate::Predicates; use sail_python_udf::error::PyErrExtractor; -use sail_server::actor::{ActorAction, ActorContext}; use tokio::sync::oneshot; use tokio::time::Instant; @@ -28,22 +27,8 @@ use crate::stream::writer::{LocalStreamStorage, TaskStreamSink}; use crate::task::scheduling::{TaskAssignment, TaskAssignmentGetter, TaskStreamAssignment}; impl DriverActor { - pub(super) fn handle_server_ready( - &mut self, - ctx: &mut ActorContext, - port: u16, - signal: oneshot::Sender<()>, - ) -> ActorAction { - let server = mem::take(&mut self.server); - self.server = match server.ready(signal) { - Ok(x) => x, - Err(e) => { - error!("{e}"); - return ActorAction::Stop; - } - }; - info!("driver server is ready on port {port}"); - self.worker_pool.set_driver_server_port(port); + pub(super) fn handle_activate(&mut self, ctx: &mut ActorContext) -> ActorAction { + info!("activating driver {}", self.options.driver_id); for _ in 0..self.options.worker_initial_count { self.worker_pool.start_worker(ctx); } @@ -318,12 +303,15 @@ impl DriverActor { pub(super) fn handle_create_remote_stream( &mut self, _ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> ActorAction { - let _ = result.send(self.stream_manager.create_remote_stream(uri, key, schema)); + let _ = result.send( + self.stream_manager + .create_remote_stream(key, schema, &context), + ); ActorAction::Continue } @@ -355,14 +343,14 @@ impl DriverActor { pub(super) fn handle_fetch_remote_stream( &mut self, ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> ActorAction { let _ = result.send( self.stream_manager - .fetch_remote_stream(ctx, uri, &key, schema), + .fetch_remote_stream(ctx, &key, schema, &context), ); ActorAction::Continue } @@ -424,12 +412,12 @@ impl DriverActor { pub(super) fn handle_shutdown( &mut self, _ctx: &mut ActorContext, - history: Option>, + result: Option>, ) -> ActorAction { - if self.history.is_some() { - warn!("overriding existing history sender"); + if self.shutdown_notifier.is_some() { + warn!("overriding existing shutdown notifier"); } - self.history = history; + self.shutdown_notifier = result; ActorAction::Stop } @@ -512,10 +500,14 @@ impl DriverActor { handle.send(JobOutputItem::Error { cause }).await; }); } - JobAction::CleanUpJob { job_id, stage } => { + JobAction::CleanUpJob { + job_id, + stage, + context, + } => { if self.task_assigner.untrack_remote_streams(job_id, stage) { self.stream_manager - .remove_remote_streams(ctx, job_id, stage); + .remove_remote_streams(ctx, job_id, stage, context); } for x in self.task_assigner.untrack_local_streams(job_id, stage) { match x { diff --git a/crates/sail-execution/src/driver/actor/mod.rs b/crates/sail-execution/src/driver/actor/mod.rs index 070ad9569..49b0647eb 100644 --- a/crates/sail-execution/src/driver/actor/mod.rs +++ b/crates/sail-execution/src/driver/actor/mod.rs @@ -1,22 +1,20 @@ mod core; mod handler; -mod rpc; use std::collections::HashMap; -use sail_common_datafusion::session::job::JobRunnerHistory; +use sail_common_datafusion::session::job::JobRunnerHistoryReporter; use tokio::sync::oneshot; use crate::driver::job_scheduler::JobScheduler; use crate::driver::task_assigner::TaskAssigner; use crate::id::TaskKey; -use crate::rpc::ServerMonitor; use crate::stream_manager::StreamManager; use crate::task_runner::TaskRunner; pub struct DriverActor { options: super::options::DriverOptions, - server: ServerMonitor, + history_reporter: Box, worker_pool: super::worker_pool::WorkerPool, job_scheduler: JobScheduler, task_assigner: TaskAssigner, @@ -25,6 +23,6 @@ pub struct DriverActor { /// The sequence number corresponding to the last task status update from the worker. /// A different sequence number is tracked for each attempt. task_sequences: HashMap, - /// An optional channel to send history when stopping the driver. - history: Option>, + /// An optional channel to signal that the driver has stopped. + shutdown_notifier: Option>, } diff --git a/crates/sail-execution/src/driver/actor/rpc.rs b/crates/sail-execution/src/driver/actor/rpc.rs deleted file mode 100644 index 0d058367e..000000000 --- a/crates/sail-execution/src/driver/actor/rpc.rs +++ /dev/null @@ -1,79 +0,0 @@ -use arrow_flight::flight_service_server::FlightServiceServer; -use sail_common::config::GRPC_MAX_MESSAGE_LENGTH_DEFAULT; -use sail_server::actor::ActorHandle; -use sail_server::ServerBuilder; -use tokio::net::{TcpListener, ToSocketAddrs}; -use tokio::sync::oneshot::Sender; -use tonic::async_trait; -use tonic::codec::CompressionEncoding; - -use crate::driver::actor::DriverActor; -use crate::driver::gen::driver_service_server::DriverServiceServer; -use crate::driver::server::DriverServer; -use crate::driver::DriverEvent; -use crate::error::{ExecutionError, ExecutionResult}; -use crate::id::TaskStreamKey; -use crate::stream::reader::TaskStreamSource; -use crate::stream_service::{TaskStreamFetcher, TaskStreamFlightServer}; - -struct DriverTaskStreamFetcher { - handle: ActorHandle, -} - -#[async_trait] -impl TaskStreamFetcher for DriverTaskStreamFetcher { - async fn fetch( - &self, - key: TaskStreamKey, - sender: Sender>, - ) -> ExecutionResult<()> { - let event = DriverEvent::FetchDriverStream { - key, - result: sender, - }; - self.handle.send(event).await.map_err(ExecutionError::from) - } -} - -impl DriverActor { - pub(super) async fn serve( - handle: ActorHandle, - addr: impl ToSocketAddrs, - ) -> ExecutionResult<()> { - let listener = TcpListener::bind(addr).await?; - let port = listener.local_addr()?.port(); - let (tx, rx) = tokio::sync::oneshot::channel(); - let server = DriverServer::new(handle.clone()); - let service = DriverServiceServer::new(server) - .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT) - .accept_compressed(CompressionEncoding::Gzip) - .accept_compressed(CompressionEncoding::Zstd) - .send_compressed(CompressionEncoding::Gzip) - .send_compressed(CompressionEncoding::Zstd); - - let flight_server = TaskStreamFlightServer::new(Box::new(DriverTaskStreamFetcher { - handle: handle.clone(), - })); - let flight_service = FlightServiceServer::new(flight_server) - .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT) - .accept_compressed(CompressionEncoding::Gzip) - .accept_compressed(CompressionEncoding::Zstd) - .send_compressed(CompressionEncoding::Gzip) - .send_compressed(CompressionEncoding::Zstd); - - handle - .send(DriverEvent::ServerReady { port, signal: tx }) - .await?; - - ServerBuilder::new("sail_driver", Default::default()) - .add_service(service, Some(crate::driver::gen::FILE_DESCRIPTOR_SET)) - .await - .add_service(flight_service, None) - .await - .serve(listener, async { - let _ = rx.await; - }) - .await - .map_err(|e| ExecutionError::InternalError(e.to_string())) - } -} diff --git a/crates/sail-execution/src/driver/client.rs b/crates/sail-execution/src/driver/client.rs index 159647a27..2d772d01b 100644 --- a/crates/sail-execution/src/driver/client.rs +++ b/crates/sail-execution/src/driver/client.rs @@ -1,16 +1,16 @@ use sail_common_datafusion::error::CommonErrorCause; use crate::driver::event::TaskStatus; -use crate::driver::gen; -use crate::driver::gen::driver_service_client::DriverServiceClient; -use crate::driver::gen::{ +use crate::driver::r#gen; +use crate::driver::r#gen::driver_service_client::DriverServiceClient; +use crate::driver::r#gen::{ RegisterWorkerRequest, RegisterWorkerResponse, ReportTaskStatusRequest, ReportTaskStatusResponse, }; use crate::error::{ExecutionError, ExecutionResult}; -use crate::id::{TaskKey, WorkerId}; +use crate::id::{DriverId, TaskKey, WorkerId}; use crate::rpc::{ClientHandle, ClientOptions, ClientService}; -use crate::stream_service::TaskStreamFlightClient; +use crate::stream_service::{TaskStreamFlightClient, TaskStreamOwner}; #[derive(Clone)] pub struct DriverClientSet { @@ -19,10 +19,10 @@ pub struct DriverClientSet { } impl DriverClientSet { - pub fn new(options: ClientOptions) -> Self { + pub fn new(driver_id: DriverId, options: ClientOptions) -> Self { Self { - core: DriverClient::new(options.clone()), - flight: TaskStreamFlightClient::new(options), + core: DriverClient::new(driver_id, options.clone()), + flight: TaskStreamFlightClient::new(options, TaskStreamOwner::Driver { driver_id }), } } } @@ -30,12 +30,14 @@ impl DriverClientSet { #[derive(Clone)] pub struct DriverClient { inner: ClientHandle>, + driver_id: DriverId, } impl DriverClient { - pub fn new(options: ClientOptions) -> Self { + pub fn new(driver_id: DriverId, options: ClientOptions) -> Self { Self { inner: ClientHandle::new(options.clone()), + driver_id, } } @@ -46,6 +48,7 @@ impl DriverClient { port: u16, ) -> ExecutionResult<()> { let request = tonic::Request::new(RegisterWorkerRequest { + driver_id: self.driver_id.into(), worker_id: worker_id.into(), host, port: port as u32, @@ -56,7 +59,8 @@ impl DriverClient { } pub async fn report_worker_heartbeat(&self, worker_id: WorkerId) -> ExecutionResult<()> { - let request = tonic::Request::new(gen::ReportWorkerHeartbeatRequest { + let request = tonic::Request::new(r#gen::ReportWorkerHeartbeatRequest { + driver_id: self.driver_id.into(), worker_id: worker_id.into(), }); let response = self @@ -65,7 +69,7 @@ impl DriverClient { .await? .report_worker_heartbeat(request) .await?; - let gen::ReportWorkerHeartbeatResponse {} = response.into_inner(); + let r#gen::ReportWorkerHeartbeatResponse {} = response.into_inner(); Ok(()) } @@ -74,7 +78,8 @@ impl DriverClient { worker_id: WorkerId, peer_worker_ids: Vec, ) -> ExecutionResult<()> { - let request = tonic::Request::new(gen::ReportWorkerKnownPeersRequest { + let request = tonic::Request::new(r#gen::ReportWorkerKnownPeersRequest { + driver_id: self.driver_id.into(), worker_id: worker_id.into(), peer_worker_ids: peer_worker_ids.into_iter().map(|id| id.into()).collect(), }); @@ -84,7 +89,7 @@ impl DriverClient { .await? .report_worker_known_peers(request) .await?; - let gen::ReportWorkerKnownPeersResponse {} = response.into_inner(); + let r#gen::ReportWorkerKnownPeersResponse {} = response.into_inner(); Ok(()) } @@ -104,11 +109,12 @@ impl DriverClient { }) .transpose()?; let request = tonic::Request::new(ReportTaskStatusRequest { + driver_id: self.driver_id.into(), job_id: key.job_id.into(), stage: key.stage as u64, partition: key.partition as u64, attempt: key.attempt as u64, - status: gen::TaskStatus::from(status) as i32, + status: r#gen::TaskStatus::from(status) as i32, message, cause, sequence, diff --git a/crates/sail-execution/src/driver/event.rs b/crates/sail-execution/src/driver/event.rs index becaa585e..1fabbd4c7 100644 --- a/crates/sail-execution/src/driver/event.rs +++ b/crates/sail-execution/src/driver/event.rs @@ -6,26 +6,20 @@ use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::ExecutionPlan; +use sail_common::telemetry::{SpanAssociation, SpanAttribute}; use sail_common_datafusion::error::CommonErrorCause; -use sail_common_datafusion::session::job::JobRunnerHistory; use sail_common_datafusion::system::observable::JobRunnerObserver; -use sail_telemetry::common::{SpanAssociation, SpanAttribute}; use tokio::sync::oneshot; use tokio::time::Instant; -use crate::driver::gen; +use crate::driver::r#gen; use crate::error::ExecutionResult; use crate::id::{JobId, TaskKey, TaskStreamKey, WorkerId}; use crate::stream::reader::TaskStreamSource; use crate::stream::writer::{LocalStreamStorage, TaskStreamSink}; pub enum DriverEvent { - ServerReady { - /// The local port that the driver server listens on. - /// This may be different from the port accessible from other nodes. - port: u16, - signal: oneshot::Sender<()>, - }, + Activate, RegisterWorker { worker_id: WorkerId, host: String, @@ -80,9 +74,9 @@ pub enum DriverEvent { result: oneshot::Sender>>, }, CreateRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, }, FetchDriverStream { @@ -96,16 +90,16 @@ pub enum DriverEvent { result: oneshot::Sender>, }, FetchRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, }, ObserveState { observer: JobRunnerObserver, }, Shutdown { - history: Option>, + result: Option>, }, } @@ -129,24 +123,24 @@ impl fmt::Display for TaskStatus { } } -impl From for TaskStatus { - fn from(value: gen::TaskStatus) -> Self { +impl From for TaskStatus { + fn from(value: r#gen::TaskStatus) -> Self { match value { - gen::TaskStatus::Running => Self::Running, - gen::TaskStatus::Succeeded => Self::Succeeded, - gen::TaskStatus::Failed => Self::Failed, - gen::TaskStatus::Canceled => Self::Canceled, + r#gen::TaskStatus::Running => Self::Running, + r#gen::TaskStatus::Succeeded => Self::Succeeded, + r#gen::TaskStatus::Failed => Self::Failed, + r#gen::TaskStatus::Canceled => Self::Canceled, } } } -impl From for gen::TaskStatus { +impl From for r#gen::TaskStatus { fn from(value: TaskStatus) -> Self { match value { - TaskStatus::Running => gen::TaskStatus::Running, - TaskStatus::Succeeded => gen::TaskStatus::Succeeded, - TaskStatus::Failed => gen::TaskStatus::Failed, - TaskStatus::Canceled => gen::TaskStatus::Canceled, + TaskStatus::Running => r#gen::TaskStatus::Running, + TaskStatus::Succeeded => r#gen::TaskStatus::Succeeded, + TaskStatus::Failed => r#gen::TaskStatus::Failed, + TaskStatus::Canceled => r#gen::TaskStatus::Canceled, } } } @@ -154,7 +148,7 @@ impl From for gen::TaskStatus { impl SpanAssociation for DriverEvent { fn name(&self) -> Cow<'static, str> { let name = match self { - DriverEvent::ServerReady { .. } => "ServerReady", + DriverEvent::Activate => "Activate", DriverEvent::RegisterWorker { .. } => "RegisterWorker", DriverEvent::WorkerHeartbeat { .. } => "WorkerHeartbeat", DriverEvent::WorkerKnownPeers { .. } => "WorkerKnownPeers", @@ -180,9 +174,7 @@ impl SpanAssociation for DriverEvent { fn properties(&self) -> impl IntoIterator, Cow<'static, str>)> { let mut p: Vec<(&'static str, String)> = vec![]; match self { - DriverEvent::ServerReady { port, signal: _ } => { - p.push((SpanAttribute::CLUSTER_DRIVER_PORT, port.to_string())); - } + DriverEvent::Activate => {} DriverEvent::RegisterWorker { worker_id, host, @@ -299,7 +291,6 @@ impl SpanAssociation for DriverEvent { )); } DriverEvent::CreateRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -309,6 +300,7 @@ impl SpanAssociation for DriverEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -316,7 +308,6 @@ impl SpanAssociation for DriverEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } DriverEvent::FetchDriverStream { key: @@ -356,7 +347,6 @@ impl SpanAssociation for DriverEvent { p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); } DriverEvent::FetchRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -366,6 +356,7 @@ impl SpanAssociation for DriverEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -373,7 +364,6 @@ impl SpanAssociation for DriverEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } DriverEvent::ObserveState { observer: _ } => {} DriverEvent::Shutdown { .. } => {} diff --git a/crates/sail-execution/src/driver/gateway.rs b/crates/sail-execution/src/driver/gateway.rs new file mode 100644 index 000000000..3c11bea3b --- /dev/null +++ b/crates/sail-execution/src/driver/gateway.rs @@ -0,0 +1,158 @@ +use std::sync::Arc; + +use arrow_flight::flight_service_server::FlightServiceServer; +use prost::Message; +use sail_common::config::{AppConfig, GRPC_MAX_MESSAGE_LENGTH_DEFAULT}; +use sail_common::server::ServerBuilder; +use tokio::net::TcpListener; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::codec::CompressionEncoding; +use tonic::{Status, async_trait}; + +use crate::driver::r#gen::driver_service_server::DriverServiceServer; +use crate::driver::server::DriverServer; +use crate::driver::{DriverEvent, DriverRegistryAccessor}; +use crate::error::{ExecutionError, ExecutionResult}; +use crate::id::{DriverId, TaskStreamKey}; +use crate::stream::r#gen::{DriverTaskStreamTicket, TaskStreamTicket}; +use crate::stream::reader::TaskStreamSource; +use crate::stream_service::{TaskStreamFetcher, TaskStreamFlightServer, TaskStreamKeyDecoder}; + +enum DriverGatewayState { + Pending { + listener: TcpListener, + }, + Running { + signal: oneshot::Sender<()>, + handle: JoinHandle>, + }, + Stopped, +} + +#[derive(Clone)] +pub struct DriverGatewayOptions { + pub listen_host: String, + pub listen_port: u16, +} + +impl DriverGatewayOptions { + pub fn new(config: &AppConfig) -> Self { + Self { + listen_host: config.cluster.driver_listen_host.clone(), + listen_port: config.cluster.driver_listen_port, + } + } +} + +struct DriverTaskStreamFetcher { + registry: Arc, +} + +#[derive(Debug)] +struct DriverTaskStreamKey { + driver_id: DriverId, + stream: TaskStreamKey, +} + +impl TaskStreamKeyDecoder for DriverTaskStreamKey { + fn decode(bytes: &[u8]) -> Result { + let DriverTaskStreamTicket { driver_id, stream } = DriverTaskStreamTicket::decode(bytes) + .map_err(|e| Status::invalid_argument(e.to_string()))?; + let stream: TaskStreamTicket = + stream.ok_or_else(|| Status::invalid_argument("task stream ticket is required"))?; + Ok(Self { + driver_id: driver_id.into(), + stream: stream.into(), + }) + } +} + +#[async_trait] +impl TaskStreamFetcher for DriverTaskStreamFetcher { + async fn fetch( + &self, + key: DriverTaskStreamKey, + sender: oneshot::Sender>, + ) -> ExecutionResult<()> { + self.registry + .get(key.driver_id) + .await? + .send(DriverEvent::FetchDriverStream { + key: key.stream, + result: sender, + }) + .await + .map_err(ExecutionError::from) + } +} + +pub struct DriverGateway { + port: u16, + state: DriverGatewayState, +} + +impl DriverGateway { + pub async fn try_new(options: DriverGatewayOptions) -> ExecutionResult { + let listener = TcpListener::bind((options.listen_host, options.listen_port)).await?; + let port = listener.local_addr()?.port(); + Ok(Self { + port, + state: DriverGatewayState::Pending { listener }, + }) + } + + pub fn start(&mut self, registry: Arc) { + let DriverGatewayState::Pending { listener } = + std::mem::replace(&mut self.state, DriverGatewayState::Stopped) + else { + return; + }; + let (tx, rx) = oneshot::channel(); + + let service = DriverServiceServer::new(DriverServer::new(registry.clone())) + .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT) + .accept_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Zstd) + .send_compressed(CompressionEncoding::Gzip) + .send_compressed(CompressionEncoding::Zstd); + let flight_service = + FlightServiceServer::new(TaskStreamFlightServer::::new( + Box::new(DriverTaskStreamFetcher { registry }), + )) + .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT) + .accept_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Zstd) + .send_compressed(CompressionEncoding::Gzip) + .send_compressed(CompressionEncoding::Zstd); + + let handle = tokio::spawn(async move { + ServerBuilder::new("sail_driver", Default::default()) + .add_service(service, Some(crate::driver::r#gen::FILE_DESCRIPTOR_SET)) + .await + .add_service(flight_service, None) + .await + .serve(listener, async { + let _ = rx.await; + }) + .await + .map_err(|e| ExecutionError::InternalError(e.to_string())) + }); + self.state = DriverGatewayState::Running { signal: tx, handle }; + } + + /// The local port that the driver server listens on. + /// This may be different from the port accessible from other nodes. + pub fn port(&self) -> u16 { + self.port + } + + pub async fn stop(&mut self) { + if let DriverGatewayState::Running { signal, handle } = + std::mem::replace(&mut self.state, DriverGatewayState::Stopped) + { + let _ = signal.send(()); + let _ = handle.await; + } + } +} diff --git a/crates/sail-execution/src/driver/job_scheduler/core.rs b/crates/sail-execution/src/driver/job_scheduler/core.rs index f77930330..e9a602fc4 100644 --- a/crates/sail-execution/src/driver/job_scheduler/core.rs +++ b/crates/sail-execution/src/driver/job_scheduler/core.rs @@ -7,23 +7,23 @@ use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use indexmap::{IndexMap, IndexSet}; use log::{debug, warn}; +use sail_common::actor::ActorContext; use sail_common_datafusion::error::CommonErrorCause; use sail_python_udf::error::PyErrExtractor; -use sail_server::actor::ActorContext; +use crate::driver::DriverActor; use crate::driver::job_scheduler::state::{ JobDescriptor, JobState, StageState, TaskAttemptDescriptor, TaskRegionState, TaskState, }; use crate::driver::job_scheduler::topology::TaskRegionTopology; use crate::driver::job_scheduler::{JobAction, JobScheduler, JobSchedulerOptions}; use crate::driver::output::build_job_output; -use crate::driver::DriverActor; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::{JobId, TaskKey, TaskKeyDisplay, TaskStreamKey}; use crate::job_graph::{ InputMode, JobGraph, OutputDistribution, OutputMode, Stage, StageInput, TaskPlacement, }; -use crate::proto::encode::{try_encode_physical_expr, try_encode_physical_plan}; +use crate::proto::{encode_remote_physical_expr, encode_remote_physical_plan}; use crate::task::definition::{ TaskDefinition, TaskInput, TaskInputKey, TaskInputLocator, TaskOutput, TaskOutputDistribution, TaskOutputLocator, @@ -34,7 +34,7 @@ use crate::task::scheduling::{ impl JobScheduler { fn next_job_id(&mut self) -> ExecutionResult { - self.job_id_generator.next() + self.job_id_generator.generate() } pub fn accept_job( @@ -49,11 +49,16 @@ impl JobScheduler { "job {job_id} execution plan\n{}", DisplayableExecutionPlan::new(plan.as_ref()).indent(true) ); - let graph = JobGraph::try_new(plan)?; + let graph = JobGraph::try_new( + plan, + crate::job_graph::JobGraphOptions { + use_blocking_shuffle: self.options.use_blocking_shuffle, + }, + )?; debug!("job {job_id} job graph \n{graph}"); let (output, stream) = build_job_output(ctx, job_id, graph.schema().clone()); - let descriptor = JobDescriptor::try_new(graph, JobState::Running { output, context })?; + let descriptor = JobDescriptor::try_new(graph, JobState::Running { output }, context)?; self.jobs.insert(job_id, descriptor); Ok((job_id, stream)) @@ -169,12 +174,11 @@ impl JobScheduler { for (r, region) in job.topology.regions.iter().enumerate() { let failed = region.tasks.iter().any(|t| { let attempts = &job.stages[t.stage].tasks[t.partition].attempts; - if let Some(attempt) = attempts.last() { - if matches!(attempt.state, TaskState::Failed | TaskState::Canceled) - && attempts.len() >= options.task_max_attempts - { - return true; - } + if let Some(attempt) = attempts.last() + && matches!(attempt.state, TaskState::Failed | TaskState::Canceled) + && attempts.len() >= options.task_max_attempts + { + return true; } false }); @@ -204,10 +208,10 @@ impl JobScheduler { for t in ®ion.tasks { let attempts = &job.stages[t.stage].tasks[t.partition].attempts; - if let Some(attempt) = attempts.last() { - if matches!(attempt.state, TaskState::Failed) { - failed = true; - } + if let Some(attempt) = attempts.last() + && matches!(attempt.state, TaskState::Failed) + { + failed = true; } } @@ -262,6 +266,7 @@ impl JobScheduler { actions.push(JobAction::CleanUpJob { job_id, stage: Some(s), + context: job.context.clone(), }); } } @@ -431,10 +436,10 @@ impl JobScheduler { for (s, stage) in job.stages.iter().enumerate() { for (t, task) in stage.tasks.iter().enumerate() { for attempt in task.attempts.iter() { - if matches!(attempt.state, TaskState::Failed) { - if let Some(cause) = &attempt.cause { - causes.entry((s, t)).or_default().push(cause); - } + if matches!(attempt.state, TaskState::Failed) + && let Some(cause) = &attempt.cause + { + causes.entry((s, t)).or_default().push(cause); } } } @@ -490,6 +495,7 @@ impl JobScheduler { actions.push(JobAction::CleanUpJob { job_id, stage: None, + context: job.context.clone(), }); if matches!(job.state, JobState::Draining) { job.state = JobState::Succeeded; @@ -512,12 +518,6 @@ impl JobScheduler { key.job_id ))); }; - let JobState::Running { context, .. } = &job.state else { - return Err(ExecutionError::InvalidArgument(format!( - "job {} is not running", - key.job_id - ))); - }; let Some(stage) = job.graph.stages().get(key.stage) else { return Err(ExecutionError::InvalidArgument(format!( "stage {} not found in job {}", @@ -525,7 +525,7 @@ impl JobScheduler { ))); }; - let plan = try_encode_physical_plan(self.codec.as_ref(), stage.plan.clone())?; + let plan = encode_remote_physical_plan(self.codec.as_ref(), stage.plan.clone())?; let inputs = stage .inputs .iter() @@ -537,7 +537,7 @@ impl JobScheduler { inputs, output, }; - Ok((definition, context.clone())) + Ok((definition, job.context.clone())) } pub fn stop(&mut self) { @@ -653,14 +653,10 @@ impl JobScheduler { } } }, - OutputMode::Blocking => { - let uri = Err(ExecutionError::InternalError("not implemented".to_string()))?; - TaskInputLocator::Remote { - uri, - stage: input.stage, - keys, - } - } + OutputMode::Blocking => TaskInputLocator::Remote { + stage: input.stage, + keys, + }, }; Ok(TaskInput { locator }) } @@ -677,7 +673,7 @@ impl JobScheduler { let keys = keys .iter() .map(|expr| { - let expr = try_encode_physical_expr(self.codec.as_ref(), expr)?; + let expr = encode_remote_physical_expr(self.codec.as_ref(), expr)?; Ok(Arc::from(expr)) }) .collect::>>>()?; @@ -697,10 +693,7 @@ impl JobScheduler { }; let locator = match stage.mode { OutputMode::Pipelined => TaskOutputLocator::Local { replicas }, - OutputMode::Blocking => { - let uri = Err(ExecutionError::InternalError("not implemented".to_string()))?; - TaskOutputLocator::Remote { uri } - } + OutputMode::Blocking => TaskOutputLocator::Remote, }; Ok(TaskOutput { distribution, diff --git a/crates/sail-execution/src/driver/job_scheduler/mod.rs b/crates/sail-execution/src/driver/job_scheduler/mod.rs index da25f7aea..82d0a8812 100644 --- a/crates/sail-execution/src/driver/job_scheduler/mod.rs +++ b/crates/sail-execution/src/driver/job_scheduler/mod.rs @@ -4,7 +4,10 @@ mod options; mod state; mod topology; +use std::sync::Arc; + use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use indexmap::IndexMap; pub use options::JobSchedulerOptions; @@ -14,7 +17,7 @@ pub use state::TaskState; use crate::driver::job_scheduler::state::JobDescriptor; use crate::driver::output::JobOutputHandle; use crate::id::{IdGenerator, JobId, TaskKey, TaskStreamKey}; -use crate::proto::codec::RemoteExecutionCodec; +use crate::proto::RemoteExecutionCodec; use crate::task::scheduling::TaskRegion; pub struct JobScheduler { @@ -55,5 +58,6 @@ pub enum JobAction { CleanUpJob { job_id: JobId, stage: Option, + context: Arc, }, } diff --git a/crates/sail-execution/src/driver/job_scheduler/options.rs b/crates/sail-execution/src/driver/job_scheduler/options.rs index 47edebff9..1612081bf 100644 --- a/crates/sail-execution/src/driver/job_scheduler/options.rs +++ b/crates/sail-execution/src/driver/job_scheduler/options.rs @@ -8,6 +8,7 @@ use crate::driver::DriverOptions; pub struct JobSchedulerOptions { pub task_launch_timeout: Duration, pub task_max_attempts: usize, + pub use_blocking_shuffle: bool, } impl From<&DriverOptions> for JobSchedulerOptions { @@ -15,6 +16,10 @@ impl From<&DriverOptions> for JobSchedulerOptions { Self { task_launch_timeout: options.task_launch_timeout, task_max_attempts: options.task_max_attempts, + use_blocking_shuffle: matches!( + &options.shuffle_backend, + crate::shuffle::ShuffleBackendKind::Storage { .. } + ), } } } diff --git a/crates/sail-execution/src/driver/job_scheduler/state.rs b/crates/sail-execution/src/driver/job_scheduler/state.rs index f744a3ad1..122b99eb0 100644 --- a/crates/sail-execution/src/driver/job_scheduler/state.rs +++ b/crates/sail-execution/src/driver/job_scheduler/state.rs @@ -14,6 +14,7 @@ use crate::job_graph::JobGraph; /// Tracks graph/topology and runtime state for a single job. pub struct JobDescriptor { + pub context: Arc, pub graph: JobGraph, pub topology: JobTopology, pub stages: Vec, @@ -24,10 +25,7 @@ pub struct JobDescriptor { } pub enum JobState { - Running { - output: JobOutputManager, - context: Arc, - }, + Running { output: JobOutputManager }, Draining, Succeeded, Failed, @@ -47,7 +45,11 @@ impl JobState { } impl JobDescriptor { - pub fn try_new(graph: JobGraph, state: JobState) -> ExecutionResult { + pub fn try_new( + graph: JobGraph, + state: JobState, + context: Arc, + ) -> ExecutionResult { let mut stages = vec![]; for stage in graph.stages().iter() { let mut descriptor = StageDescriptor { @@ -68,6 +70,7 @@ impl JobDescriptor { }) .collect(); Ok(Self { + context, graph, topology, stages, diff --git a/crates/sail-execution/src/driver/job_scheduler/topology.rs b/crates/sail-execution/src/driver/job_scheduler/topology.rs index 802b04677..fb9496a04 100644 --- a/crates/sail-execution/src/driver/job_scheduler/topology.rs +++ b/crates/sail-execution/src/driver/job_scheduler/topology.rs @@ -176,10 +176,9 @@ impl JobTopology { if let Some(&d) = task_to_region.get(&TaskTopology { stage: input.stage, partition: task.partition, - }) { - if d != r { - region.dependencies.insert(d); - } + }) && d != r + { + region.dependencies.insert(d); } } else { let partitions = graph.stages()[input.stage] @@ -190,10 +189,9 @@ impl JobTopology { if let Some(&d) = task_to_region.get(&TaskTopology { stage: input.stage, partition: p, - }) { - if d != r { - region.dependencies.insert(d); - } + }) && d != r + { + region.dependencies.insert(d); } } } diff --git a/crates/sail-execution/src/driver/mod.rs b/crates/sail-execution/src/driver/mod.rs index 208dacdef..a198fc475 100644 --- a/crates/sail-execution/src/driver/mod.rs +++ b/crates/sail-execution/src/driver/mod.rs @@ -1,15 +1,17 @@ mod actor; mod client; mod event; +mod gateway; pub(super) mod job_scheduler; mod options; pub(super) mod output; +mod registry; mod server; mod task_assigner; pub(super) mod worker_pool; #[expect(clippy::allow_attributes)] -mod gen { +mod r#gen { tonic::include_proto!("sail.driver"); pub const FILE_DESCRIPTOR_SET: &[u8] = @@ -19,5 +21,7 @@ mod gen { pub(crate) use actor::DriverActor; pub(crate) use client::DriverClientSet; pub(crate) use event::{DriverEvent, TaskStatus}; -pub(crate) use gen::driver_service_client::DriverServiceClient; -pub use options::DriverOptions; +pub use gateway::{DriverGateway, DriverGatewayOptions}; +pub(crate) use r#gen::driver_service_client::DriverServiceClient; +pub use options::{DriverComponents, DriverOptions}; +pub use registry::{DriverHandle, DriverRegistry, DriverRegistryAccessor}; diff --git a/crates/sail-execution/src/driver/options.rs b/crates/sail-execution/src/driver/options.rs index b4d617b5d..713fa2250 100644 --- a/crates/sail-execution/src/driver/options.rs +++ b/crates/sail-execution/src/driver/options.rs @@ -1,17 +1,20 @@ -use std::sync::Arc; use std::time::Duration; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeHandle; -use sail_server::RetryStrategy; +use sail_common::utils::retry::RetryStrategy; +use sail_common_datafusion::session::job::JobRunnerHistoryReporter; +use crate::id::DriverId; +use crate::shuffle::ShuffleBackendKind; use crate::worker_manager::WorkerManager; #[readonly::make] pub struct DriverOptions { pub enable_tls: bool, - pub driver_listen_host: String, - pub driver_listen_port: u16, + pub session_id: String, + pub driver_id: DriverId, + pub driver_server_port: u16, pub driver_external_host: String, pub driver_external_port: u16, pub worker_initial_count: usize, @@ -25,21 +28,29 @@ pub struct DriverOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub task_max_attempts: usize, + pub shuffle_backend: ShuffleBackendKind, pub rpc_retry_strategy: RetryStrategy, pub runtime: RuntimeHandle, - pub worker_manager: Arc, +} + +pub struct DriverComponents { + pub worker_manager: Box, + pub history_reporter: Box, } impl DriverOptions { pub fn new( config: &AppConfig, runtime: RuntimeHandle, - worker_manager: Arc, + session_id: String, + driver_id: DriverId, + driver_server_port: u16, ) -> Self { Self { enable_tls: config.cluster.enable_tls, - driver_listen_host: config.cluster.driver_listen_host.clone(), - driver_listen_port: config.cluster.driver_listen_port, + session_id, + driver_id, + driver_server_port, driver_external_host: config.cluster.driver_external_host.clone(), driver_external_port: config.cluster.driver_external_port, worker_initial_count: config.cluster.worker_initial_count, @@ -60,8 +71,8 @@ impl DriverOptions { config.cluster.task_stream_creation_timeout_secs, ), task_max_attempts: config.cluster.task_max_attempts, + shuffle_backend: (&config.cluster.shuffle_backend).into(), runtime, - worker_manager, } } } diff --git a/crates/sail-execution/src/driver/output.rs b/crates/sail-execution/src/driver/output.rs index 0e0a33be6..20174e7dd 100644 --- a/crates/sail-execution/src/driver/output.rs +++ b/crates/sail-execution/src/driver/output.rs @@ -9,8 +9,8 @@ use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use futures::stream::SelectAll; use futures::{Stream, StreamExt}; +use sail_common::actor::ActorContext; use sail_common_datafusion::error::CommonErrorCause; -use sail_server::actor::ActorContext; use tokio::sync::mpsc; use tonic::codegen::tokio_stream::wrappers::ReceiverStream; diff --git a/crates/sail-execution/src/driver/registry.rs b/crates/sail-execution/src/driver/registry.rs new file mode 100644 index 000000000..4514731b5 --- /dev/null +++ b/crates/sail-execution/src/driver/registry.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; + +use sail_common::actor::ActorHandle; +use tokio::sync::mpsc::error::SendError; +use tokio::sync::oneshot; +use tonic::async_trait; + +use crate::driver::{DriverActor, DriverEvent}; +use crate::error::{ExecutionError, ExecutionResult}; +use crate::id::DriverId; + +/// A handle for managing a driver actor. +/// +/// This wrapper lets the session manager own the driver lifecycle without exposing +/// [`DriverActor`] or [`DriverEvent`] outside the `sail-execution` crate. +/// Keeping the underlying actor handle private prevents callers from sending arbitrary +/// driver events and avoids coupling session management to the driver actor implementation. +#[derive(Clone)] +pub struct DriverHandle { + handle: ActorHandle, +} + +impl DriverHandle { + pub(crate) fn new(handle: ActorHandle) -> Self { + Self { handle } + } + + pub(crate) async fn send(&self, event: DriverEvent) -> Result<(), SendError> { + self.handle.send(event).await + } + + pub async fn activate(&self) -> ExecutionResult<()> { + self.send(DriverEvent::Activate) + .await + .map_err(ExecutionError::from) + } + + pub async fn shutdown(&self) -> ExecutionResult<()> { + // A closed channel means that the driver actor has already stopped. + // Shutdown is intentionally idempotent, so this is still a success. + let _ = self.send(DriverEvent::Shutdown { result: None }).await; + Ok(()) + } + + /// Stop the driver and wait for its shutdown hook to complete. + pub async fn shutdown_and_wait(&self) -> ExecutionResult<()> { + let (tx, rx) = oneshot::channel(); + if self + .send(DriverEvent::Shutdown { result: Some(tx) }) + .await + .is_ok() + { + // A closed result channel means another shutdown request won the race. + // In either case, the driver actor is no longer running. + let _ = rx.await; + } + Ok(()) + } +} + +#[derive(Default)] +pub struct DriverRegistry { + drivers: HashMap, +} + +impl DriverRegistry { + pub fn insert(&mut self, driver_id: DriverId, handle: DriverHandle) -> ExecutionResult<()> { + if self.drivers.contains_key(&driver_id) { + return Err(ExecutionError::InternalError(format!( + "driver {driver_id} is already registered" + ))); + } + self.drivers.insert(driver_id, handle); + Ok(()) + } + + pub fn remove(&mut self, driver_id: DriverId) -> Option { + self.drivers.remove(&driver_id) + } + + pub fn get(&self, driver_id: DriverId) -> ExecutionResult { + self.drivers + .get(&driver_id) + .cloned() + .ok_or_else(|| ExecutionError::InvalidArgument(format!("driver {driver_id} not found"))) + } + + pub fn drain(&mut self) -> impl Iterator + '_ { + self.drivers.drain() + } +} + +#[async_trait] +pub trait DriverRegistryAccessor: Send + Sync { + async fn get(&self, driver_id: DriverId) -> ExecutionResult; +} diff --git a/crates/sail-execution/src/driver/server.rs b/crates/sail-execution/src/driver/server.rs index 1fbf8e9ef..0b65da802 100644 --- a/crates/sail-execution/src/driver/server.rs +++ b/crates/sail-execution/src/driver/server.rs @@ -1,26 +1,26 @@ +use std::sync::Arc; + use log::debug; -use sail_server::actor::ActorHandle; use tokio::sync::oneshot; use tonic::{Request, Response, Status}; -use crate::driver::actor::DriverActor; -use crate::driver::gen::driver_service_server::DriverService; -use crate::driver::gen::{ +use crate::driver::r#gen::driver_service_server::DriverService; +use crate::driver::r#gen::{ RegisterWorkerRequest, RegisterWorkerResponse, ReportTaskStatusRequest, ReportTaskStatusResponse, ReportWorkerHeartbeatRequest, ReportWorkerHeartbeatResponse, ReportWorkerKnownPeersRequest, ReportWorkerKnownPeersResponse, }; -use crate::driver::{gen, DriverEvent}; +use crate::driver::{DriverEvent, DriverRegistryAccessor, r#gen}; use crate::error::ExecutionError; -use crate::id::{TaskKey, WorkerId}; +use crate::id::{DriverId, TaskKey, WorkerId}; pub struct DriverServer { - handle: ActorHandle, + registry: Arc, } impl DriverServer { - pub fn new(handle: ActorHandle) -> Self { - Self { handle } + pub fn new(registry: Arc) -> Self { + Self { registry } } } @@ -33,6 +33,7 @@ impl DriverService for DriverServer { let request = request.into_inner(); debug!("{request:?}"); let RegisterWorkerRequest { + driver_id, worker_id, host, port, @@ -47,7 +48,9 @@ impl DriverService for DriverServer { port, result: tx, }; - self.handle + self.registry + .get(DriverId::from(driver_id)) + .await? .send(event) .await .map_err(ExecutionError::from)?; @@ -63,11 +66,16 @@ impl DriverService for DriverServer { ) -> Result, Status> { let request = request.into_inner(); debug!("{request:?}"); - let ReportWorkerHeartbeatRequest { worker_id } = request; + let ReportWorkerHeartbeatRequest { + driver_id, + worker_id, + } = request; let event = DriverEvent::WorkerHeartbeat { worker_id: worker_id.into(), }; - self.handle + self.registry + .get(DriverId::from(driver_id)) + .await? .send(event) .await .map_err(ExecutionError::from)?; @@ -83,6 +91,7 @@ impl DriverService for DriverServer { let request = request.into_inner(); debug!("{request:?}"); let ReportWorkerKnownPeersRequest { + driver_id, worker_id, peer_worker_ids, } = request; @@ -90,7 +99,9 @@ impl DriverService for DriverServer { worker_id: worker_id.into(), peer_worker_ids: peer_worker_ids.into_iter().map(|x| x.into()).collect(), }; - self.handle + self.registry + .get(DriverId::from(driver_id)) + .await? .send(event) .await .map_err(ExecutionError::from)?; @@ -106,6 +117,7 @@ impl DriverService for DriverServer { let request = request.into_inner(); debug!("{request:?}"); let ReportTaskStatusRequest { + driver_id, job_id, stage, partition, @@ -115,7 +127,7 @@ impl DriverService for DriverServer { cause, sequence, } = request; - let status = gen::TaskStatus::try_from(status).map_err(ExecutionError::from)?; + let status = r#gen::TaskStatus::try_from(status).map_err(ExecutionError::from)?; let cause = cause .map(|x| serde_json::from_str(&x)) .transpose() @@ -132,7 +144,9 @@ impl DriverService for DriverServer { cause, sequence: Some(sequence), }; - self.handle + self.registry + .get(DriverId::from(driver_id)) + .await? .send(event) .await .map_err(ExecutionError::from)?; diff --git a/crates/sail-execution/src/driver/worker_pool/core.rs b/crates/sail-execution/src/driver/worker_pool/core.rs index cedb7db3e..18abb24d7 100644 --- a/crates/sail-execution/src/driver/worker_pool/core.rs +++ b/crates/sail-execution/src/driver/worker_pool/core.rs @@ -3,14 +3,14 @@ use std::sync::Arc; use chrono::Utc; use datafusion::arrow::datatypes::SchemaRef; -use fastrace::collector::SpanContext; use fastrace::Span; +use fastrace::collector::SpanContext; use futures::TryStreamExt; use log::{error, info, warn}; +use sail_common::actor::ActorContext; +use sail_common::telemetry::SpanAttribute; use sail_common_datafusion::error::CommonErrorCause; use sail_python_udf::error::PyErrExtractor; -use sail_server::actor::ActorContext; -use sail_telemetry::common::SpanAttribute; use tokio::time::Instant; use crate::driver::worker_pool::state::WorkerState; @@ -29,7 +29,6 @@ impl WorkerPool { pub async fn close(&mut self, ctx: &mut ActorContext) -> ExecutionResult<()> { let worker_ids = self.workers.keys().cloned().collect::>(); for worker_id in worker_ids.into_iter() { - // TODO: Should we wait for the spawned tasks for stopping the workers? self.stop_worker(ctx, worker_id, Some("closing worker pool".to_string())); } // TODO: support timeout for worker manager stop @@ -37,14 +36,10 @@ impl WorkerPool { Ok(()) } - pub fn set_driver_server_port(&mut self, port: u16) { - self.driver_server_port = Some(port); - } - pub fn start_worker(&mut self, ctx: &mut ActorContext) { - let Ok(worker_id) = self.worker_id_generator.next() else { + let Ok(worker_id) = self.worker_id_generator.generate() else { error!("failed to generate worker ID"); - ctx.send(DriverEvent::Shutdown { history: None }); + ctx.send(DriverEvent::Shutdown { result: None }); return; }; let descriptor = WorkerDescriptor { @@ -70,26 +65,27 @@ impl WorkerPool { let span = Span::root("WorkerPool::start_worker", SpanContext::random()) .with_property(|| (SpanAttribute::CLUSTER_WORKER_ID, worker_id.to_string())); let _guard = span.set_local_parent(); - let Some(port) = self.driver_server_port else { - error!("the driver server is not ready"); - return; - }; let options = WorkerLaunchOptions { enable_tls: self.options.enable_tls, + session_id: self.options.session_id.clone(), + driver_id: self.options.driver_id, driver_external_host: self.options.driver_external_host.to_string(), driver_external_port: if self.options.driver_external_port > 0 { self.options.driver_external_port } else { - port + self.options.driver_server_port }, worker_heartbeat_interval: self.options.worker_heartbeat_interval, task_stream_buffer: self.options.task_stream_buffer, task_stream_creation_timeout: self.options.task_stream_creation_timeout, rpc_retry_strategy: self.options.rpc_retry_strategy.clone(), + shuffle_backend: self.options.shuffle_backend.clone(), }; - let worker_manager = Arc::clone(&self.worker_manager); + let task = self + .worker_manager + .launch_worker(ctx.children_mut(), worker_id, options); ctx.spawn(async move { - if let Err(e) = worker_manager.launch_worker(worker_id, options).await { + if let Err(e) = task.await { error!("failed to start worker {worker_id}: {e}"); } }); diff --git a/crates/sail-execution/src/driver/worker_pool/mod.rs b/crates/sail-execution/src/driver/worker_pool/mod.rs index ea748336c..69dc2608a 100644 --- a/crates/sail-execution/src/driver/worker_pool/mod.rs +++ b/crates/sail-execution/src/driver/worker_pool/mod.rs @@ -14,18 +14,16 @@ use crate::worker_manager::WorkerManager; pub struct WorkerPool { options: WorkerPoolOptions, - driver_server_port: Option, worker_manager: Arc, workers: IndexMap, worker_id_generator: IdGenerator, } impl WorkerPool { - pub fn new(worker_manager: Arc, options: WorkerPoolOptions) -> Self { + pub fn new(worker_manager: Box, options: WorkerPoolOptions) -> Self { Self { options, - driver_server_port: None, - worker_manager, + worker_manager: Arc::from(worker_manager), workers: IndexMap::new(), worker_id_generator: IdGenerator::new(), } diff --git a/crates/sail-execution/src/driver/worker_pool/options.rs b/crates/sail-execution/src/driver/worker_pool/options.rs index 8dff30544..29b087ef5 100644 --- a/crates/sail-execution/src/driver/worker_pool/options.rs +++ b/crates/sail-execution/src/driver/worker_pool/options.rs @@ -1,13 +1,18 @@ //! A dedicated module for worker pool options to ensure readonly access. use std::time::Duration; -use sail_server::RetryStrategy; +use sail_common::utils::retry::RetryStrategy; use crate::driver::DriverOptions; +use crate::id::DriverId; +use crate::shuffle::ShuffleBackendKind; #[readonly::make] pub struct WorkerPoolOptions { pub enable_tls: bool, + pub session_id: String, + pub driver_id: DriverId, + pub driver_server_port: u16, pub driver_external_host: String, pub driver_external_port: u16, pub worker_max_idle_time: Duration, @@ -17,12 +22,16 @@ pub struct WorkerPoolOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackendKind, } impl From<&DriverOptions> for WorkerPoolOptions { fn from(options: &DriverOptions) -> Self { Self { enable_tls: options.enable_tls, + session_id: options.session_id.clone(), + driver_id: options.driver_id, + driver_server_port: options.driver_server_port, driver_external_host: options.driver_external_host.clone(), driver_external_port: options.driver_external_port, worker_max_idle_time: options.worker_max_idle_time, @@ -32,6 +41,7 @@ impl From<&DriverOptions> for WorkerPoolOptions { task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, rpc_retry_strategy: options.rpc_retry_strategy.clone(), + shuffle_backend: options.shuffle_backend.clone(), } } } diff --git a/crates/sail-execution/src/id.rs b/crates/sail-execution/src/id.rs index ea967e6d5..ed6562b1c 100644 --- a/crates/sail-execution/src/id.rs +++ b/crates/sail-execution/src/id.rs @@ -64,6 +64,7 @@ macro_rules! define_id_type { } define_id_type!(JobId, u64); +define_id_type!(DriverId, u64); define_id_type!(WorkerId, u64); #[derive(Debug)] @@ -83,13 +84,22 @@ where } } - pub fn next(&mut self) -> ExecutionResult { + pub fn generate(&mut self) -> ExecutionResult { let value = self.next_value; self.next_value = T::Value::next(value)?; Ok(value.into()) } } +impl Default for IdGenerator +where + T::Value: Copy, +{ + fn default() -> Self { + Self::new() + } +} + /// Uniquely identifies a task attempt within a job by stage, partition, and attempt number. #[derive(Debug, Clone, Eq, Hash, PartialEq)] pub struct TaskKey { diff --git a/crates/sail-execution/src/job_graph/mod.rs b/crates/sail-execution/src/job_graph/mod.rs index cdefa9b11..a944cdd35 100644 --- a/crates/sail-execution/src/job_graph/mod.rs +++ b/crates/sail-execution/src/job_graph/mod.rs @@ -8,6 +8,11 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +#[derive(Debug, Clone)] +pub struct JobGraphOptions { + pub use_blocking_shuffle: bool, +} + /// A job graph represents a distributed execution plan for a job. /// A job consists of multiple *stages*, where each stage has one or more /// *partitions*. There are *tasks* which each corresponds to the execution of a single partition @@ -21,6 +26,7 @@ pub struct JobGraph { stages: Vec, /// The output schema of the job. schema: SchemaRef, + options: JobGraphOptions, } impl JobGraph { @@ -165,7 +171,6 @@ impl fmt::Display for InputMode { #[derive(Debug, Clone, Copy)] pub enum OutputMode { Pipelined, - #[expect(unused)] Blocking, } diff --git a/crates/sail-execution/src/job_graph/planner.rs b/crates/sail-execution/src/job_graph/planner.rs index 71f1d1a29..053fad1e5 100644 --- a/crates/sail-execution/src/job_graph/planner.rs +++ b/crates/sail-execution/src/job_graph/planner.rs @@ -1,40 +1,58 @@ use std::sync::Arc; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; -use datafusion::common::{plan_datafusion_err, JoinType, Result}; +use datafusion::common::{JoinType, Result, plan_datafusion_err}; +use datafusion::logical_expr::execution_props::ScalarSubqueryResults; +use datafusion::physical_expr::scalar_subquery::ScalarSubqueryExpr; use datafusion::physical_expr::{Partitioning, PhysicalExpr}; +use datafusion::physical_plan::aggregates::AggregateExec; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::coop::CooperativeExec; +use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::joins::{ CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode, PiecewiseMergeJoinExec, }; use datafusion::physical_plan::limit::GlobalLimitExec; +use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink}; +use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec}; use datafusion::physical_plan::{ - with_new_children_if_necessary, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + ExecutionPlan, ExecutionPlanProperties, PlanProperties, with_new_children_if_necessary, }; use sail_catalog_system::physical_plan::SystemTableExec; use sail_common_datafusion::utils::items::ItemTaker; use sail_data_source::listing::delete::FileDeleteExec; +use sail_delta_lake::physical_plan::DeltaCommitExec; +use sail_iceberg::physical_plan::IcebergCommitExec; +use sail_physical_plan::barrier::BarrierExec; use sail_physical_plan::catalog_command::CatalogCommandExec; use sail_physical_plan::coalesce::CoalesceExec; +use sail_physical_plan::remote_checkpoint::RemoteCheckpointCommitExec; use sail_physical_plan::repartition::ExplicitRepartitionExec; use crate::error::{ExecutionError, ExecutionResult}; use crate::job_graph::{ - InputMode, JobGraph, OutputDistribution, OutputMode, Stage, StageInput, TaskPlacement, + InputMode, JobGraph, JobGraphOptions, OutputDistribution, OutputMode, Stage, StageInput, + TaskPlacement, }; use crate::plan::{ShuffleConsumption, StageInputExec}; impl JobGraph { - pub fn try_new(plan: Arc) -> ExecutionResult { + pub fn try_new( + plan: Arc, + options: JobGraphOptions, + ) -> ExecutionResult { let plan = ensure_single_input_partition_for_global_limit(plan)?; let plan = ensure_partitioned_hash_join_if_build_side_emits_unmatched_rows(plan)?; let mut graph = Self { stages: vec![], schema: plan.schema(), + options, }; - let last = build_job_graph(plan, PartitionUsage::Once, &mut graph)?; + let last = build_job_graph(plan, PartitionUsage::Once, &mut graph)?.plan; let (last, inputs) = rewrite_inputs(last)?; graph.stages.push(Stage { inputs, @@ -168,12 +186,93 @@ enum PartitionUsage { Shared, } +#[derive(Clone, Copy)] +enum DriverStageHandling { + CreateStage, + PreserveRoot, +} + +#[derive(Clone, Copy)] +struct ScalarSubqueryContext<'a> { + links: &'a [ScalarSubqueryLink], + results: &'a ScalarSubqueryResults, +} + +struct PlannedSubtree { + plan: Arc, + // TODO: Track pending SubqueryIndex values if wrapper placement must + // distinguish multiple or nested scalar subqueries. + has_pending_scalar_subquery_expr: bool, +} + +impl PlannedSubtree { + fn new(plan: Arc, has_pending_scalar_subquery_expr: bool) -> Self { + Self { + plan, + has_pending_scalar_subquery_expr, + } + } + + fn without_pending_scalar_subquery_expr(plan: Arc) -> Self { + Self::new(plan, false) + } +} + +struct RebuiltSubtree { + plan: Arc, + // TODO: Extend this with other stage-boundary metadata if materialization + // needs to preserve more than pending scalar subquery state. + child_has_pending_scalar_subquery_expr: Vec, + node_has_scalar_subquery_expr: bool, + subtree_has_pending_scalar_subquery_expr: bool, +} + +impl RebuiltSubtree { + fn only_child(&self) -> ExecutionResult { + let [has_pending_scalar_subquery_expr] = + self.child_has_pending_scalar_subquery_expr.as_slice() + else { + return Err(ExecutionError::InternalError(format!( + "expected exactly one planned child, got {}", + self.child_has_pending_scalar_subquery_expr.len() + ))); + }; + let child = self.plan.children().one()?; + Ok(PlannedSubtree::new( + child.clone(), + *has_pending_scalar_subquery_expr, + )) + } + + fn into_planned_subtree(self) -> PlannedSubtree { + PlannedSubtree::new(self.plan, self.subtree_has_pending_scalar_subquery_expr) + } +} + /// Recursively splits an execution plan into stages at shuffle boundaries and adds them to the job graph. fn build_job_graph( plan: Arc, usage: PartitionUsage, graph: &mut JobGraph, -) -> ExecutionResult> { +) -> ExecutionResult { + plan_job_graph_stages(plan, usage, graph, DriverStageHandling::CreateStage, None) +} + +fn plan_job_graph_stages( + plan: Arc, + usage: PartitionUsage, + graph: &mut JobGraph, + driver_stage_handling: DriverStageHandling, + scalar_context: Option>, +) -> ExecutionResult { + if let Some(scalar) = plan.downcast_ref::() { + return build_scalar_subquery_job_graph(scalar, usage, graph, driver_stage_handling); + } + + if let Some(barrier) = plan.downcast_ref::() { + return build_barrier_job_graph(barrier, usage, graph, scalar_context); + } + // Recursively build the job graph for the children first // and propagate partition usage information. let children = if let Some(join) = plan.downcast_ref::() { @@ -181,14 +280,38 @@ fn build_job_graph( match join.mode { PartitionMode::Partitioned => { vec![ - build_job_graph(left.clone(), usage, graph)?, - build_job_graph(right.clone(), usage, graph)?, + plan_job_graph_stages( + left.clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?, + plan_job_graph_stages( + right.clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?, ] } PartitionMode::CollectLeft => { vec![ - build_job_graph(left.clone(), PartitionUsage::Shared, graph)?, - build_job_graph(right.clone(), usage, graph)?, + plan_job_graph_stages( + left.clone(), + PartitionUsage::Shared, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?, + plan_job_graph_stages( + right.clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?, ] } PartitionMode::Auto => { @@ -203,8 +326,20 @@ fn build_job_graph( { let (left, right) = plan.children().two()?; vec![ - build_job_graph(left.clone(), PartitionUsage::Shared, graph)?, - build_job_graph(right.clone(), usage, graph)?, + plan_job_graph_stages( + left.clone(), + PartitionUsage::Shared, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?, + plan_job_graph_stages( + right.clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?, ] } else if plan.is::() || plan.is::() @@ -215,20 +350,46 @@ fn build_job_graph( let child = plan.children().one()?; // At the stage boundary, we only expect to use the child partition once // since the shuffle writer can materialize the data for multiple consumption. - vec![build_job_graph(child.clone(), PartitionUsage::Once, graph)?] + vec![plan_job_graph_stages( + child.clone(), + PartitionUsage::Once, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )?] + } else if matches!(driver_stage_handling, DriverStageHandling::PreserveRoot) + && plan.is::() + && is_driver_stage_plan(&plan) + { + let child = plan.children().one()?; + vec![plan_job_graph_stages( + child.clone(), + usage, + graph, + DriverStageHandling::PreserveRoot, + scalar_context, + )?] } else { plan.children() .into_iter() - .map(|x| build_job_graph(x.clone(), usage, graph)) + .map(|x| { + plan_job_graph_stages( + x.clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + ) + }) .collect::>>()? }; - let plan = with_new_children_if_necessary(plan, children)?; + let subtree = rebuild_subtree(plan, children)?; let consumption = match usage { PartitionUsage::Once => ShuffleConsumption::Single, PartitionUsage::Shared => ShuffleConsumption::Multiple, }; - let plan = if let Some(repartition) = plan.downcast_ref::() { + let plan = if let Some(repartition) = subtree.plan.downcast_ref::() { if repartition.preserve_order() { // We haven't found a case when order-preserving repartition can be constructed, // so it's fine to return an error for now. @@ -238,8 +399,8 @@ fn build_job_graph( )); } let properties = repartition.properties().clone(); - let child = plan.children().one()?; - match &properties.partitioning { + let child = subtree.only_child()?; + let plan = match &properties.partitioning { Partitioning::UnknownPartitioning(n) => { let n = *n; let properties = Arc::new( @@ -248,117 +409,402 @@ fn build_job_graph( .clone() .with_partitioning(Partitioning::RoundRobinBatch(n)), ); - create_shuffle(child, graph, properties, consumption)? + create_shuffle(child, graph, properties, consumption, scalar_context)? } Partitioning::RoundRobinBatch(_) | Partitioning::Hash(_, _) => { - create_shuffle(child, graph, properties, consumption)? + create_shuffle(child, graph, properties, consumption, scalar_context)? } - } - } else if let Some(repartition) = plan.downcast_ref::() { + }; + PlannedSubtree::without_pending_scalar_subquery_expr(plan) + } else if let Some(repartition) = subtree.plan.downcast_ref::() { let properties = repartition.properties().clone(); - let child = plan.children().one()?; - match &properties.partitioning { - Partitioning::RoundRobinBatch(channels) => { - create_row_shuffle(child, *channels, graph, properties, consumption)? - } + let child = subtree.only_child()?; + let plan = match &properties.partitioning { + Partitioning::RoundRobinBatch(channels) => create_row_shuffle( + child, + *channels, + graph, + properties, + consumption, + scalar_context, + )?, other => { return Err(ExecutionError::DataFusionError(plan_datafusion_err!( "unexpected explicit repartition partitioning in distributed planning: {other:?}" ))); } - } - } else if let Some(coalesce) = plan.downcast_ref::() { + }; + PlannedSubtree::without_pending_scalar_subquery_expr(plan) + } else if let Some(coalesce) = subtree.plan.downcast_ref::() { let properties = coalesce.properties().clone(); - let child = plan.children().one()?; let fetch = coalesce.fetch(); - let shuffled = create_shuffle(child, graph, properties, consumption)?; + let child = subtree.only_child()?; + let shuffled = create_shuffle(child, graph, properties, consumption, scalar_context)?; if let Some(f) = fetch { - Arc::new(GlobalLimitExec::new(shuffled, 0, Some(f))) as Arc + PlannedSubtree::without_pending_scalar_subquery_expr(Arc::new(GlobalLimitExec::new( + shuffled, + 0, + Some(f), + )) + as Arc) } else { - shuffled + PlannedSubtree::without_pending_scalar_subquery_expr(shuffled) } - } else if plan.is::() { - let child = plan.children().one()?; - plan.clone() - .with_new_children(vec![create_merge_input(child, graph)?])? - } else if let Some(coalesce) = plan.downcast_ref::() { - let child = plan.children().one()?; - create_rescale_input(child, coalesce.output_partitions(), graph)? - } else if plan.is::() - || plan.is::() - || plan.is::() + } else if subtree.plan.is::() { + let child = subtree.only_child()?; + let plan = subtree + .plan + .clone() + .with_new_children(vec![create_merge_input(child, graph, scalar_context)?])?; + PlannedSubtree::new(plan, subtree.node_has_scalar_subquery_expr) + } else if let Some(coalesce) = subtree.plan.downcast_ref::() { + let child = subtree.only_child()?; + let plan = + create_rescale_input(child, coalesce.output_partitions(), graph, scalar_context)?; + PlannedSubtree::without_pending_scalar_subquery_expr(plan) + } else if subtree.plan.is::() + || subtree.plan.is::() + || subtree.plan.is::() + || subtree.plan.is::() + || subtree.plan.is::() + || subtree.plan.is::() { - plan.children().zero()?; - create_driver_stage(&plan, graph)? + if matches!(driver_stage_handling, DriverStageHandling::PreserveRoot) { + subtree.into_planned_subtree() + } else { + let plan = create_driver_stage(subtree.into_planned_subtree(), graph, scalar_context)?; + PlannedSubtree::without_pending_scalar_subquery_expr(plan) + } } else { - plan + subtree.into_planned_subtree() }; Ok(plan) } +fn rebuild_subtree( + plan: Arc, + children: Vec, +) -> ExecutionResult { + let child_has_pending_scalar_subquery_expr = children + .iter() + .map(|child| child.has_pending_scalar_subquery_expr) + .collect::>(); + let children = children + .into_iter() + .map(|child| child.plan) + .collect::>(); + let plan = with_new_children_if_necessary(plan, children)?; + let node_has_scalar_subquery_expr = plan_node_has_scalar_subquery_expr(&plan); + let subtree_has_pending_scalar_subquery_expr = + node_has_scalar_subquery_expr || child_has_pending_scalar_subquery_expr.iter().any(|x| *x); + Ok(RebuiltSubtree { + plan, + child_has_pending_scalar_subquery_expr, + node_has_scalar_subquery_expr, + subtree_has_pending_scalar_subquery_expr, + }) +} + +fn build_scalar_subquery_job_graph( + scalar: &ScalarSubqueryExec, + usage: PartitionUsage, + graph: &mut JobGraph, + driver_stage_handling: DriverStageHandling, +) -> ExecutionResult { + let subqueries = scalar + .subqueries() + .iter() + .map(|link| { + // The link plan is materialized once as its own stage. Reuse happens + // at the outer broadcast input consumed by ScalarSubqueryExec. + let plan = plan_job_graph_stages( + link.plan.clone(), + PartitionUsage::Once, + graph, + DriverStageHandling::CreateStage, + None, + )?; + let plan = create_scalar_subquery_input(&plan.plan, graph)?; + Ok(ScalarSubqueryLink { + plan, + index: link.index, + }) + }) + .collect::>>()?; + let input = plan_job_graph_stages( + scalar.input().clone(), + usage, + graph, + driver_stage_handling, + Some(ScalarSubqueryContext { + links: subqueries.as_slice(), + results: scalar.results(), + }), + )?; + let plan = wrap_pending_scalar_subqueries( + input, + Some(ScalarSubqueryContext { + links: subqueries.as_slice(), + results: scalar.results(), + }), + ); + Ok(PlannedSubtree::without_pending_scalar_subquery_expr(plan)) +} + +fn build_barrier_job_graph( + barrier: &BarrierExec, + usage: PartitionUsage, + graph: &mut JobGraph, + scalar_context: Option>, +) -> ExecutionResult { + let preconditions = barrier + .preconditions() + .iter() + .map(|precondition| { + plan_job_graph_stages( + precondition.clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + ) + }) + .collect::>>()?; + let preconditions_have_pending_scalar_subquery_expr = preconditions + .iter() + .any(|precondition| precondition.has_pending_scalar_subquery_expr); + let preconditions = preconditions + .into_iter() + .map(|precondition| precondition.plan) + .collect::>(); + let plan_is_driver_stage = is_driver_stage_plan(barrier.plan()); + let plan = if plan_is_driver_stage { + plan_job_graph_stages( + barrier.plan().clone(), + usage, + graph, + DriverStageHandling::PreserveRoot, + scalar_context, + )? + } else { + plan_job_graph_stages( + barrier.plan().clone(), + usage, + graph, + DriverStageHandling::CreateStage, + scalar_context, + )? + }; + let barrier_has_pending_scalar_subquery_expr = + preconditions_have_pending_scalar_subquery_expr || plan.has_pending_scalar_subquery_expr; + let barrier = Arc::new(BarrierExec::new(preconditions, plan.plan)) as Arc; + let barrier = PlannedSubtree::new(barrier, barrier_has_pending_scalar_subquery_expr); + if plan_is_driver_stage { + Ok(PlannedSubtree::without_pending_scalar_subquery_expr( + create_driver_stage(barrier, graph, scalar_context)?, + )) + } else { + Ok(barrier) + } +} + +fn is_driver_stage_plan(plan: &Arc) -> bool { + if let Some(cooperative) = plan.downcast_ref::() { + return is_driver_stage_plan(cooperative.input()); + } + + plan.is::() + || plan.is::() + || plan.is::() + || plan.is::() + || plan.is::() + || plan.is::() +} + +fn wrap_pending_scalar_subqueries( + plan: PlannedSubtree, + scalar_context: Option>, +) -> Arc { + let Some(scalar_context) = scalar_context else { + return plan.plan; + }; + if scalar_context.links.is_empty() || !plan.has_pending_scalar_subquery_expr { + return plan.plan; + } + Arc::new(ScalarSubqueryExec::new( + plan.plan, + scalar_context.links.to_vec(), + scalar_context.results.clone(), + )) +} + +fn plan_node_has_scalar_subquery_expr(plan: &Arc) -> bool { + if let Some(filter) = plan.downcast_ref::() { + return physical_expr_has_scalar_subquery(filter.predicate()); + } + if let Some(projection) = plan.downcast_ref::() { + return projection + .expr() + .iter() + .any(|expr| physical_expr_has_scalar_subquery(&expr.expr)); + } + if let Some(aggregate) = plan.downcast_ref::() { + return aggregate_has_scalar_subquery_expr(aggregate); + } + if let Some(sort) = plan.downcast_ref::() { + return sort + .expr() + .iter() + .any(|sort| physical_expr_has_scalar_subquery(&sort.expr)); + } + if let Some(sort) = plan.downcast_ref::() { + return sort + .expr() + .iter() + .any(|sort| physical_expr_has_scalar_subquery(&sort.expr)); + } + if let Some(join) = plan.downcast_ref::() { + return hash_join_has_scalar_subquery_expr(join); + } + if let Some(join) = plan.downcast_ref::() { + return join + .filter() + .is_some_and(|filter| physical_expr_has_scalar_subquery(filter.expression())); + } + if let Some(join) = plan.downcast_ref::() { + return physical_expr_has_scalar_subquery(&join.on.0) + || physical_expr_has_scalar_subquery(&join.on.1); + } + if let Some(window) = plan.downcast_ref::() { + return window + .window_expr() + .iter() + .any(window_expr_has_scalar_subquery); + } + if let Some(window) = plan.downcast_ref::() { + return window + .window_expr() + .iter() + .any(window_expr_has_scalar_subquery); + } + false +} + +fn aggregate_has_scalar_subquery_expr(aggregate: &AggregateExec) -> bool { + aggregate + .group_expr() + .expr() + .iter() + .any(|(expr, _)| physical_expr_has_scalar_subquery(expr)) + || aggregate + .group_expr() + .null_expr() + .iter() + .any(|(expr, _)| physical_expr_has_scalar_subquery(expr)) + || aggregate.aggr_expr().iter().any(|expr| { + expr.expressions() + .iter() + .any(physical_expr_has_scalar_subquery) + || expr + .order_bys() + .iter() + .any(|sort| physical_expr_has_scalar_subquery(&sort.expr)) + }) + || aggregate + .filter_expr() + .iter() + .flatten() + .any(physical_expr_has_scalar_subquery) +} + +fn hash_join_has_scalar_subquery_expr(join: &HashJoinExec) -> bool { + join.on().iter().any(|(left, right)| { + physical_expr_has_scalar_subquery(left) || physical_expr_has_scalar_subquery(right) + }) || join + .filter() + .is_some_and(|filter| physical_expr_has_scalar_subquery(filter.expression())) +} + +fn window_expr_has_scalar_subquery( + expr: &Arc, +) -> bool { + let expressions = expr.all_expressions(); + expressions + .args + .iter() + .chain(expressions.partition_by_exprs.iter()) + .chain(expressions.order_by_exprs.iter()) + .any(physical_expr_has_scalar_subquery) +} + +fn physical_expr_has_scalar_subquery(expr: &Arc) -> bool { + expr.downcast_ref::().is_some() + || expr + .children() + .iter() + .any(|child| physical_expr_has_scalar_subquery(child)) +} + fn create_merge_input( - plan: &Arc, + plan: PlannedSubtree, graph: &mut JobGraph, + scalar_context: Option>, ) -> ExecutionResult> { - let properties = plan.properties().clone(); - let (plan, inputs) = rewrite_inputs(plan.clone())?; - let stage = Stage { - inputs, + let properties = plan.plan.properties().clone(); + let plan = wrap_pending_scalar_subqueries(plan, scalar_context); + let stage = push_stage( plan, - group: String::new(), - mode: OutputMode::Pipelined, - distribution: OutputDistribution::RoundRobin { channels: 1 }, - placement: TaskPlacement::Worker, - }; - let s = graph.stages.len(); - graph.stages.push(stage); - Ok(Arc::new(StageInputExec::new( - StageInput { - stage: s, - mode: InputMode::Merge, - }, - properties, - ))) + graph, + OutputDistribution::RoundRobin { channels: 1 }, + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; + Ok(stage_input_exec(stage, InputMode::Merge, properties)) } -fn create_rescale_input( +fn create_scalar_subquery_input( plan: &Arc, + graph: &mut JobGraph, +) -> ExecutionResult> { + let stage = push_stage( + Arc::clone(plan), + graph, + OutputDistribution::RoundRobin { channels: 1 }, + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; + // ScalarSubqueryExec reads the link as a scalar value on every output + // partition, so the materialized stage is exposed as one broadcast input. + let properties = stage_properties_with_unknown_partitioning(graph, stage, 1); + Ok(stage_input_exec(stage, InputMode::Broadcast, properties)) +} + +fn create_rescale_input( + plan: PlannedSubtree, output_partitions: usize, graph: &mut JobGraph, + scalar_context: Option>, ) -> ExecutionResult> { - let (plan, inputs) = rewrite_inputs(plan.clone())?; - let stage = Stage { - inputs, + let plan = wrap_pending_scalar_subqueries(plan, scalar_context); + let stage = push_stage( plan, - group: String::new(), - mode: OutputMode::Pipelined, - distribution: OutputDistribution::RoundRobin { channels: 1 }, - placement: TaskPlacement::Worker, - }; - let s = graph.stages.len(); - graph.stages.push(stage); - let properties = Arc::new(PlanProperties::new( - datafusion::physical_expr::EquivalenceProperties::new(graph.stages[s].plan.schema()), - Partitioning::UnknownPartitioning(output_partitions), - graph.stages[s].plan.pipeline_behavior(), - graph.stages[s].plan.boundedness(), - )); - Ok(Arc::new(StageInputExec::new( - StageInput { - stage: s, - mode: InputMode::Rescale, - }, - properties, - ))) + graph, + OutputDistribution::RoundRobin { channels: 1 }, + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; + let properties = stage_properties_with_unknown_partitioning(graph, stage, output_partitions); + Ok(stage_input_exec(stage, InputMode::Rescale, properties)) } fn create_shuffle( - plan: &Arc, + plan: PlannedSubtree, graph: &mut JobGraph, // These are the properties after repartition/coalesce, // which are different from the properties of the input plan. properties: Arc, consumption: ShuffleConsumption, + scalar_context: Option>, ) -> ExecutionResult> { let distribution = match properties.partitioning.clone() { Partitioning::RoundRobinBatch(channels) | Partitioning::UnknownPartitioning(channels) => { @@ -366,57 +812,70 @@ fn create_shuffle( } Partitioning::Hash(keys, channels) => OutputDistribution::Hash { keys, channels }, }; - let (plan, inputs) = rewrite_inputs(plan.clone())?; - let stage = Stage { - inputs, + let plan = wrap_pending_scalar_subqueries(plan, scalar_context); + let stage = push_stage( plan, - group: String::new(), - mode: OutputMode::Pipelined, + graph, distribution, - placement: TaskPlacement::Worker, - }; - let s = graph.stages.len(); - graph.stages.push(stage); + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; let mode = match consumption { ShuffleConsumption::Single => InputMode::Shuffle, ShuffleConsumption::Multiple => InputMode::Broadcast, }; - Ok(Arc::new(StageInputExec::new( - StageInput { stage: s, mode }, - properties, - ))) + create_shuffle_input(stage, mode, properties, graph) } /// Creates a shuffle stage with row-level round-robin distribution. /// Used for explicit user-requested repartition calls where individual rows /// must be distributed evenly across output partitions. fn create_row_shuffle( - plan: &Arc, + plan: PlannedSubtree, channels: usize, graph: &mut JobGraph, properties: Arc, consumption: ShuffleConsumption, + scalar_context: Option>, ) -> ExecutionResult> { let distribution = OutputDistribution::RoundRobinRow { channels }; - let (plan, inputs) = rewrite_inputs(plan.clone())?; - let stage = Stage { - inputs, + let plan = wrap_pending_scalar_subqueries(plan, scalar_context); + let stage = push_stage( plan, - group: String::new(), - mode: OutputMode::Pipelined, + graph, distribution, - placement: TaskPlacement::Worker, - }; - let s = graph.stages.len(); - graph.stages.push(stage); + TaskPlacement::Worker, + OutputMode::Pipelined, + )?; let mode = match consumption { ShuffleConsumption::Single => InputMode::Shuffle, ShuffleConsumption::Multiple => InputMode::Broadcast, }; - Ok(Arc::new(StageInputExec::new( - StageInput { stage: s, mode }, - properties, - ))) + create_shuffle_input(stage, mode, properties, graph) +} + +/// In storage shuffle mode, collect each logical shuffle partition in a separate +/// blocking stage. This turns the many task/channel streams produced by the upstream +/// stage into one (or a few size-bounded) object-storage files per output partition. +fn create_shuffle_input( + stage: usize, + mode: InputMode, + properties: Arc, + graph: &mut JobGraph, +) -> ExecutionResult> { + if !graph.options.use_blocking_shuffle { + return Ok(stage_input_exec(stage, mode, properties)); + } + + let merge = stage_input_exec(stage, mode, properties.clone()); + let stage = push_stage( + merge, + graph, + OutputDistribution::RoundRobin { channels: 1 }, + TaskPlacement::Worker, + OutputMode::Blocking, + )?; + Ok(stage_input_exec(stage, InputMode::Forward, properties)) } fn rewrite_inputs( @@ -436,46 +895,99 @@ fn rewrite_inputs( Ok((result.data()?, inputs)) } -// TODO: support driver stage with inputs -fn create_driver_stage( - plan: &Arc, +fn push_stage( + plan: Arc, graph: &mut JobGraph, -) -> ExecutionResult> { + distribution: OutputDistribution, + placement: TaskPlacement, + mode: OutputMode, +) -> ExecutionResult { + let (plan, inputs) = rewrite_inputs(plan)?; let stage = Stage { - inputs: vec![], - plan: plan.clone(), + inputs, + plan, group: String::new(), - mode: OutputMode::Pipelined, - distribution: OutputDistribution::RoundRobin { channels: 1 }, - placement: TaskPlacement::Driver, + mode, + distribution, + placement, }; - let s = graph.stages.len(); + let index = graph.stages.len(); graph.stages.push(stage); - Ok(Arc::new(StageInputExec::new( - StageInput { - stage: s, - mode: InputMode::Forward, - }, - plan.properties().clone(), - ))) + Ok(index) +} + +fn stage_input_exec( + stage: usize, + mode: InputMode, + properties: Arc, +) -> Arc { + Arc::new(StageInputExec::new(StageInput { stage, mode }, properties)) +} + +fn stage_properties_with_unknown_partitioning( + graph: &JobGraph, + stage: usize, + partitions: usize, +) -> Arc { + let plan = &graph.stages[stage].plan; + Arc::new(PlanProperties::new( + datafusion::physical_expr::EquivalenceProperties::new(plan.schema()), + Partitioning::UnknownPartitioning(partitions), + plan.pipeline_behavior(), + plan.boundedness(), + )) +} + +fn create_driver_stage( + plan: PlannedSubtree, + graph: &mut JobGraph, + scalar_context: Option>, +) -> ExecutionResult> { + let plan = wrap_pending_scalar_subqueries(plan, scalar_context); + let stage = push_stage( + plan, + graph, + OutputDistribution::RoundRobin { channels: 1 }, + TaskPlacement::Driver, + OutputMode::Pipelined, + )?; + let properties = graph.stages[stage].plan.properties().clone(); + Ok(stage_input_exec(stage, InputMode::Forward, properties)) } #[cfg(test)] -#[expect(clippy::unwrap_used)] +#[expect(clippy::expect_used, clippy::unwrap_used)] mod tests { use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; - use datafusion::physical_expr::Partitioning; + use datafusion::execution::object_store::ObjectStoreUrl; + use datafusion::functions_aggregate::sum::sum_udaf; + use datafusion::logical_expr::Operator; + use datafusion::logical_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex}; + use datafusion::physical_expr::aggregate::AggregateExprBuilder; + use datafusion::physical_expr::expressions::{binary, col}; + use datafusion::physical_expr::scalar_subquery::ScalarSubqueryExpr; + use datafusion::physical_expr::{Partitioning, PhysicalExpr}; + use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; + use datafusion::physical_plan::coop::CooperativeExec; use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink}; use datafusion::physical_plan::union::UnionExec; - use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; + use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable}; + use sail_catalog::command::CatalogCommand; + use sail_physical_plan::barrier::BarrierExec; + use sail_physical_plan::catalog_command::CatalogCommandExec; use sail_physical_plan::coalesce::CoalesceExec; + use sail_physical_plan::remote_checkpoint::RemoteCheckpointCommitExec; use sail_physical_plan::repartition::ExplicitRepartitionExec; - use super::JobGraph; - use crate::job_graph::{InputMode, OutputDistribution, StageInput}; + use super::{JobGraph, JobGraphOptions}; + use crate::error::ExecutionResult; + use crate::job_graph::{InputMode, OutputDistribution, OutputMode, StageInput, TaskPlacement}; + use crate::plan::StageInputExec; fn schema() -> SchemaRef { Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])) @@ -485,13 +997,28 @@ mod tests { Arc::new(EmptyExec::new(schema())) } + fn job_graph(plan: Arc) -> ExecutionResult { + JobGraph::try_new( + plan, + JobGraphOptions { + use_blocking_shuffle: false, + }, + ) + } + + fn blocking_shuffle_options() -> JobGraphOptions { + JobGraphOptions { + use_blocking_shuffle: true, + } + } + #[test] fn test_job_graph_distinguishes_batch_and_explicit_round_robin_shuffle() { - let generic_graph = JobGraph::try_new(Arc::new( + let generic_graph = job_graph(Arc::new( RepartitionExec::try_new(empty_plan(), Partitioning::RoundRobinBatch(4)).unwrap(), )) .unwrap(); - let explicit_graph = JobGraph::try_new(Arc::new(ExplicitRepartitionExec::new( + let explicit_graph = job_graph(Arc::new(ExplicitRepartitionExec::new( empty_plan(), Partitioning::RoundRobinBatch(4), ))) @@ -524,12 +1051,41 @@ mod tests { )); } + #[test] + fn test_storage_shuffle_inserts_a_blocking_merge_stage() { + let graph = JobGraph::try_new( + Arc::new( + RepartitionExec::try_new(empty_plan(), Partitioning::RoundRobinBatch(4)).unwrap(), + ), + blocking_shuffle_options(), + ) + .unwrap(); + + assert_eq!(graph.stages().len(), 3); + assert!(matches!(graph.stages()[0].mode, OutputMode::Pipelined)); + assert!(matches!( + graph.stages()[1].inputs.as_slice(), + [StageInput { + stage: 0, + mode: InputMode::Shuffle, + }] + )); + assert!(matches!(graph.stages()[1].mode, OutputMode::Blocking)); + assert!(matches!( + graph.stages()[2].inputs.as_slice(), + [StageInput { + stage: 1, + mode: InputMode::Forward, + }] + )); + } + #[test] fn test_job_graph_uses_rescale_input_for_coalesce_exec() { let input = UnionExec::try_new(vec![empty_plan(), empty_plan(), empty_plan(), empty_plan()]) .unwrap(); - let graph = JobGraph::try_new(Arc::new(CoalesceExec::new(input, 2))).unwrap(); + let graph = job_graph(Arc::new(CoalesceExec::new(input, 2))).unwrap(); assert_eq!(graph.stages().len(), 2); assert_eq!( @@ -547,4 +1103,240 @@ mod tests { }] )); } + + #[test] + fn test_job_graph_places_remote_checkpoint_commit_on_driver() { + let schema = schema(); + let commit = Arc::new(RemoteCheckpointCommitExec::new( + empty_plan(), + "relation".to_string(), + ObjectStoreUrl::parse("memory://").unwrap(), + object_store::path::Path::from("checkpoint/session/relation"), + Arc::clone(&schema), + schema, + Partitioning::UnknownPartitioning(1), + None, + )); + + let graph = JobGraph::try_new( + commit, + JobGraphOptions { + use_blocking_shuffle: false, + }, + ) + .unwrap(); + + assert_eq!(graph.stages().len(), 2); + assert_eq!(graph.stages()[0].placement, TaskPlacement::Driver); + assert!(graph.stages()[0].plan.is::()); + assert!(matches!( + graph.stages()[1].inputs.as_slice(), + [StageInput { + stage: 0, + mode: InputMode::Forward, + }] + )); + } + + #[test] + fn test_job_graph_keeps_driver_actual_plan_inside_barrier() { + let precondition = Arc::new( + RepartitionExec::try_new(empty_plan(), Partitioning::RoundRobinBatch(1)).unwrap(), + ); + let command = Arc::new(CatalogCommandExec::new( + CatalogCommand::CurrentCatalog, + schema(), + )); + let command = Arc::new(CooperativeExec::new(command)); + let graph = job_graph(Arc::new(BarrierExec::new(vec![precondition], command))).unwrap(); + + assert_eq!(graph.stages().len(), 3); + assert_eq!(graph.stages()[1].placement, TaskPlacement::Driver); + #[expect(clippy::expect_used)] + let barrier = graph.stages()[1] + .plan + .downcast_ref::() + .expect("driver stage should contain BarrierExec"); + #[expect(clippy::expect_used)] + let cooperative = barrier + .plan() + .downcast_ref::() + .expect("barrier actual plan should preserve CooperativeExec"); + assert!(cooperative.input().is::()); + assert!(matches!( + graph.stages()[1].inputs.as_slice(), + [StageInput { + stage: 0, + mode: InputMode::Shuffle, + }] + )); + assert!(matches!( + graph.stages()[2].inputs.as_slice(), + [StageInput { + stage: 1, + mode: InputMode::Forward, + }] + )); + } + + #[test] + fn test_job_graph_wraps_scalar_subquery_stage_after_shuffle_split() { + let results = ScalarSubqueryResults::new(1); + let predicate = Arc::new(ScalarSubqueryExpr::new( + DataType::Boolean, + false, + SubqueryIndex::new(0), + results.clone(), + )); + let filtered = Arc::new(FilterExec::try_new(predicate, empty_plan()).unwrap()); + let repartitioned = + Arc::new(RepartitionExec::try_new(filtered, Partitioning::RoundRobinBatch(4)).unwrap()); + let plan = Arc::new(ScalarSubqueryExec::new( + repartitioned, + vec![ScalarSubqueryLink { + plan: empty_plan(), + index: SubqueryIndex::new(0), + }], + results, + )); + + let graph = job_graph(plan).unwrap(); + + assert!(graph.stages().len() >= 3); + let stage = graph + .stages() + .iter() + .find_map(|stage| stage.plan.downcast_ref::()) + .expect("stage containing ScalarSubqueryExpr should keep a ScalarSubqueryExec wrapper"); + assert_eq!(stage.subqueries().len(), 1); + let filter = stage + .input() + .downcast_ref::() + .expect("ScalarSubqueryExec input should be the filtered stage"); + let scalar_expr = filter + .predicate() + .downcast_ref::() + .expect("filter predicate should contain a scalar subquery expression"); + assert!(ScalarSubqueryResults::ptr_eq( + stage.results(), + scalar_expr.results() + )); + } + + #[test] + fn test_job_graph_wraps_aggregate_scalar_subquery_stage_after_shuffle_split() { + let input_schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)])); + let results = ScalarSubqueryResults::new(1); + let scalar_subquery = Arc::new(ScalarSubqueryExpr::new( + DataType::Int64, + false, + SubqueryIndex::new(0), + results.clone(), + )) as Arc; + let aggregate_arg = binary( + col("v", input_schema.as_ref()).unwrap(), + Operator::Plus, + scalar_subquery, + input_schema.as_ref(), + ) + .unwrap(); + let aggregate_expr = Arc::new( + AggregateExprBuilder::new(sum_udaf(), vec![aggregate_arg]) + .schema(input_schema.clone()) + .alias("total") + .human_display("#4") + .build() + .unwrap(), + ); + let aggregate: Arc = Arc::new( + AggregateExec::try_new( + AggregateMode::Partial, + PhysicalGroupBy::new_single(vec![]), + vec![aggregate_expr], + vec![None], + Arc::new(EmptyExec::new(input_schema.clone())), + input_schema, + ) + .unwrap(), + ); + assert!( + !displayable(aggregate.as_ref()) + .indent(false) + .to_string() + .contains("scalar_subquery("), + "aggregate display should hide the scalar subquery expression" + ); + let repartitioned = Arc::new( + RepartitionExec::try_new(aggregate, Partitioning::RoundRobinBatch(4)).unwrap(), + ); + let plan = Arc::new(ScalarSubqueryExec::new( + repartitioned, + vec![ScalarSubqueryLink { + plan: empty_plan(), + index: SubqueryIndex::new(0), + }], + results, + )); + + let graph = job_graph(plan).unwrap(); + + let stage = graph + .stages() + .iter() + .find_map(|stage| stage.plan.downcast_ref::()) + .filter(|scalar| scalar.input().is::()) + .expect("aggregate stage containing ScalarSubqueryExpr should keep a ScalarSubqueryExec wrapper"); + assert_eq!(stage.subqueries().len(), 1); + } + + #[test] + fn test_job_graph_broadcasts_scalar_subquery_stage_inputs() { + let results = ScalarSubqueryResults::new(1); + let predicate = Arc::new(ScalarSubqueryExpr::new( + DataType::Boolean, + false, + SubqueryIndex::new(0), + results.clone(), + )); + let filtered = Arc::new(FilterExec::try_new(predicate, empty_plan()).unwrap()); + let repartitioned = + Arc::new(RepartitionExec::try_new(filtered, Partitioning::RoundRobinBatch(4)).unwrap()); + let subquery = Arc::new( + RepartitionExec::try_new(empty_plan(), Partitioning::RoundRobinBatch(4)).unwrap(), + ); + let plan = Arc::new(ScalarSubqueryExec::new( + repartitioned, + vec![ScalarSubqueryLink { + plan: subquery, + index: SubqueryIndex::new(0), + }], + results, + )); + + let graph = job_graph(plan).unwrap(); + let stage = graph + .stages() + .iter() + .find(|stage| stage.plan.is::()) + .expect("stage containing ScalarSubqueryExpr should keep a ScalarSubqueryExec wrapper"); + let scalar = stage.plan.downcast_ref::().unwrap(); + let subquery_input = scalar.subqueries()[0] + .plan + .downcast_ref::>() + .expect("scalar subquery link should read from a stage input"); + + assert_eq!( + subquery_input + .properties() + .output_partitioning() + .partition_count(), + 1 + ); + assert!( + stage + .inputs + .iter() + .any(|input| matches!(input.mode, InputMode::Broadcast)) + ); + } } diff --git a/crates/sail-execution/src/job_runner.rs b/crates/sail-execution/src/job_runner.rs index 17255b54c..b8e39e68c 100644 --- a/crates/sail-execution/src/job_runner.rs +++ b/crates/sail-execution/src/job_runner.rs @@ -1,40 +1,49 @@ -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use datafusion::common::{internal_datafusion_err, internal_err, Result}; +use datafusion::common::{DataFusionError, Result, internal_datafusion_err, internal_err}; use datafusion::execution::SendableRecordBatchStream; -use datafusion::physical_plan::{execute_stream, ExecutionPlan}; +use datafusion::physical_plan::{ExecutionPlan, execute_stream}; use datafusion::prelude::SessionContext; -use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistory}; +use sail_common::actor::ActorSystem; +use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistory, JobRunnerHistoryReporter}; use sail_common_datafusion::system::observable::{JobRunnerObserver, Observer, StateObservable}; -use sail_server::actor::{ActorHandle, ActorSystem}; use sail_telemetry::telemetry::global_metrics; -use sail_telemetry::{trace_execution_plan, TracingExecOptions}; +use sail_telemetry::{TracingExecOptions, trace_execution_plan}; use tokio::sync::mpsc::error::SendError; use tokio::sync::oneshot; -use crate::driver::{DriverActor, DriverEvent, DriverOptions}; +use crate::driver::{DriverActor, DriverComponents, DriverEvent, DriverHandle, DriverOptions}; +use crate::job_graph::{JobGraph, JobGraphOptions}; +use crate::shuffle::ShuffleBackendKind; + +fn explain_job_graph(plan: Arc, use_blocking_shuffle: bool) -> Result { + JobGraph::try_new( + plan, + JobGraphOptions { + use_blocking_shuffle, + }, + ) + .map(|graph| graph.to_string()) + .map_err(|e| DataFusionError::External(Box::new(e))) +} pub struct LocalJobRunner { next_job_id: AtomicU64, stopped: AtomicBool, + history_reporter: std::sync::Mutex>>, } impl LocalJobRunner { - pub fn new() -> Self { + pub fn new(history_reporter: Box) -> Self { Self { next_job_id: AtomicU64::new(1), stopped: AtomicBool::new(false), + history_reporter: std::sync::Mutex::new(Some(history_reporter)), } } } -impl Default for LocalJobRunner { - fn default() -> Self { - Self::new() - } -} - #[tonic::async_trait] impl StateObservable for LocalJobRunner { async fn observe(&self, observer: JobRunnerObserver) { @@ -44,6 +53,10 @@ impl StateObservable for LocalJobRunner { #[tonic::async_trait] impl JobRunner for LocalJobRunner { + fn explain(&self, plan: Arc) -> Result { + explain_job_graph(plan, false) + } + async fn execute( &self, ctx: &SessionContext, @@ -64,25 +77,47 @@ impl JobRunner for LocalJobRunner { Ok(execute_stream(plan, ctx.task_ctx())?) } - async fn stop(&self, history: oneshot::Sender) { + async fn stop(&self) { self.stopped.store(true, Ordering::Relaxed); - let _ = history.send(JobRunnerHistory { - jobs: vec![], - stages: vec![], - tasks: vec![], - workers: vec![], - }); + let history_reporter = self + .history_reporter + .lock() + .ok() + .and_then(|mut history_reporter| history_reporter.take()); + if let Some(history_reporter) = history_reporter { + history_reporter + .report(JobRunnerHistory { + jobs: vec![], + stages: vec![], + tasks: vec![], + workers: vec![], + }) + .await; + } } } pub struct ClusterJobRunner { - driver: ActorHandle, + driver: DriverHandle, + shuffle_backend: ShuffleBackendKind, } impl ClusterJobRunner { - pub fn new(system: &mut ActorSystem, options: DriverOptions) -> Self { - let driver = system.spawn(options); - Self { driver } + pub fn new( + system: &mut ActorSystem, + options: DriverOptions, + components: DriverComponents, + ) -> Self { + let shuffle_backend = options.shuffle_backend.clone(); + let driver = DriverHandle::new(system.spawn::((options, components))); + Self { + driver, + shuffle_backend, + } + } + + pub fn driver(&self) -> DriverHandle { + self.driver.clone() } } @@ -103,6 +138,13 @@ impl StateObservable for ClusterJobRunner { #[tonic::async_trait] impl JobRunner for ClusterJobRunner { + fn explain(&self, plan: Arc) -> Result { + explain_job_graph( + plan, + matches!(&self.shuffle_backend, ShuffleBackendKind::Storage { .. }), + ) + } + /// Executes a plan on the cluster. This is where the cool stuff happens. async fn execute( &self, @@ -123,12 +165,7 @@ impl JobRunner for ClusterJobRunner { .map_err(|e| internal_datafusion_err!("{e}")) } - async fn stop(&self, history: oneshot::Sender) { - let _ = self - .driver - .send(DriverEvent::Shutdown { - history: Some(history), - }) - .await; + async fn stop(&self) { + let _ = self.driver.shutdown_and_wait().await; } } diff --git a/crates/sail-execution/src/lib.rs b/crates/sail-execution/src/lib.rs index 068757c9a..561ea6554 100644 --- a/crates/sail-execution/src/lib.rs +++ b/crates/sail-execution/src/lib.rs @@ -6,6 +6,7 @@ pub mod job_runner; mod plan; mod proto; mod rpc; +mod shuffle; mod stream; mod stream_accessor; mod stream_manager; @@ -15,4 +16,5 @@ mod task_runner; mod worker; pub mod worker_manager; +pub use id::{DriverId, IdGenerator}; pub use worker::entrypoint::run_worker; diff --git a/crates/sail-execution/src/plan/mod.rs b/crates/sail-execution/src/plan/mod.rs index 4b45c5f61..ebbd0e29a 100644 --- a/crates/sail-execution/src/plan/mod.rs +++ b/crates/sail-execution/src/plan/mod.rs @@ -8,7 +8,7 @@ pub(crate) use shuffle_read::ShuffleReadExec; pub(crate) use shuffle_write::ShuffleWriteExec; pub(crate) use stage_input::StageInputExec; -pub(crate) mod gen { +pub(crate) mod r#gen { tonic::include_proto!("sail.plan"); } diff --git a/crates/sail-execution/src/plan/shuffle_read.rs b/crates/sail-execution/src/plan/shuffle_read.rs index 80a85485e..4479ea1fe 100644 --- a/crates/sail-execution/src/plan/shuffle_read.rs +++ b/crates/sail-execution/src/plan/shuffle_read.rs @@ -2,12 +2,12 @@ use std::fmt::Formatter; use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::{exec_datafusion_err, internal_err, Result}; +use datafusion::common::{Result, exec_datafusion_err, internal_err}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use futures::future::try_join_all; use futures::TryStreamExt; +use futures::future::try_join_all; use log::warn; use crate::plan::ListListDisplay; diff --git a/crates/sail-execution/src/plan/shuffle_write.rs b/crates/sail-execution/src/plan/shuffle_write.rs index 0b985ff05..7eb53c911 100644 --- a/crates/sail-execution/src/plan/shuffle_write.rs +++ b/crates/sail-execution/src/plan/shuffle_write.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::{exec_datafusion_err, plan_err, Result}; +use datafusion::common::{Result, exec_datafusion_err, plan_err}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::expressions::UnKnownColumn; use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; @@ -11,11 +11,11 @@ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::repartition::BatchPartitioner; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - internal_err, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, - PlanProperties, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + internal_err, }; -use futures::future::try_join_all; use futures::StreamExt; +use futures::future::try_join_all; use sail_physical_plan::repartition::RowRoundRobinPartitioner; use crate::plan::ListListDisplay; diff --git a/crates/sail-execution/src/plan/stage_input.rs b/crates/sail-execution/src/plan/stage_input.rs index ed654b7a4..67cfbc538 100644 --- a/crates/sail-execution/src/plan/stage_input.rs +++ b/crates/sail-execution/src/plan/stage_input.rs @@ -2,7 +2,7 @@ use std::fmt; use std::fmt::Formatter; use std::sync::Arc; -use datafusion::common::{internal_err, Result}; +use datafusion::common::{Result, internal_err}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; diff --git a/crates/sail-execution/src/proto/codec.rs b/crates/sail-execution/src/proto/codec.rs index c8e291e5f..b7e95da37 100644 --- a/crates/sail-execution/src/proto/codec.rs +++ b/crates/sail-execution/src/proto/codec.rs @@ -5,16 +5,18 @@ use std::sync::Arc; use datafusion::arrow::compute::SortOptions; use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use datafusion::common::config::{CsvOptions, TableParquetOptions}; use datafusion::common::parsers::CompressionTypeVariant; use datafusion::common::{ - plan_datafusion_err, plan_err, Constraint, Constraints, JoinSide, Result, ScalarValue, - Statistics, + Constraint, Constraints, JoinSide, Result, ScalarValue, Statistics, plan_datafusion_err, + plan_err, }; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::datasource::physical_plan::parquet::CachedParquetFileReaderFactory; use datafusion::datasource::physical_plan::{ - ArrowSource, AvroSource, FileScanConfig, FileScanConfigBuilder, FileSink, FileSinkConfig, - JsonSource, + ArrowSource, AvroSource, CsvSource, FileScanConfig, FileScanConfigBuilder, FileSink, + FileSinkConfig, FileSource, JsonSource, ParquetSource, }; use datafusion::datasource::sink::DataSinkExec; use datafusion::datasource::source::{DataSource, DataSourceExec}; @@ -22,18 +24,24 @@ use datafusion::execution::TaskContext; use datafusion::functions::core::greatest::GreatestFunc; use datafusion::functions::core::least::LeastFunc; use datafusion::functions::string::overlay::OverlayFunc; +use datafusion::functions_window::cume_dist::cume_dist_udwf; +use datafusion::functions_window::lead_lag::{lag_udwf, lead_udwf}; +use datafusion::functions_window::nth_value::{first_value_udwf, last_value_udwf, nth_value_udwf}; +use datafusion::functions_window::rank::{dense_rank_udwf, percent_rank_udwf, rank_udwf}; +use datafusion::functions_window::row_number::row_number_udwf; use datafusion::logical_expr::{ AggregateUDF, AggregateUDFImpl, ScalarUDF, ScalarUDFImpl, WindowUDF, }; +use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; use datafusion::physical_expr::equivalence::{EquivalenceClass, EquivalenceGroup}; -use datafusion::physical_expr::expressions::{LambdaExpr, LambdaVariable}; +use datafusion::physical_expr::expressions::{Column, LambdaExpr, LambdaVariable}; use datafusion::physical_expr::{ AcrossPartitions, ConstExpr, EquivalenceProperties, LexOrdering, LexRequirement, Partitioning, PhysicalExpr, PhysicalSortExpr, }; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter}; use datafusion::physical_plan::joins::SortMergeJoinExec; +use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter}; use datafusion::physical_plan::recursive_query::RecursiveQueryExec; use datafusion::physical_plan::sorts::partial_sort::PartialSortExec; use datafusion::physical_plan::work_table::WorkTableExec; @@ -41,7 +49,7 @@ use datafusion::physical_plan::{ExecutionPlan, PlanProperties}; use datafusion_proto::generated::datafusion_common as gen_datafusion_common; use datafusion_proto::physical_plan::from_proto::{ parse_physical_sort_exprs, parse_protobuf_file_scan_config, parse_protobuf_file_scan_schema, - parse_protobuf_partitioning, + parse_protobuf_partitioning, parse_table_schema_from_proto, }; use datafusion_proto::physical_plan::to_proto::{ serialize_file_scan_config, serialize_partitioning, serialize_physical_sort_exprs, @@ -66,6 +74,7 @@ use datafusion_spark::function::math::hex::SparkHex; use datafusion_spark::function::math::width_bucket::SparkWidthBucket; use datafusion_spark::function::string::elt::SparkElt; use datafusion_spark::function::string::format_string::FormatStringFunc; +use datafusion_spark::function::string::length::SparkLengthFunc; use datafusion_spark::function::string::luhn_check::SparkLuhnCheck; use datafusion_spark::function::url::try_url_decode::TryUrlDecode; use datafusion_spark::function::url::url_decode::UrlDecode; @@ -77,11 +86,15 @@ use sail_common_datafusion::catalog::{ CatalogPartitionField, LakehouseExecutionContext, PartitionTransform, }; use sail_common_datafusion::datasource::PhysicalSinkMode; -use sail_common_datafusion::schema_evolution::SchemaEvolutionCastColumnExpr; +use sail_common_datafusion::schema_evolution::{ + SchemaEvolutionCastColumnExpr, SchemaEvolutionPhysicalExprAdapterFactoryWithMatching, + SchemaEvolutionTimezoneMode, StructFieldMatching, +}; use sail_common_datafusion::system::catalog::SystemTable; use sail_common_datafusion::udf::StreamUDF; use sail_data_source::formats::binary::source::BinarySource; use sail_data_source::formats::console::ConsoleSinkExec; +use sail_data_source::formats::noop::NoopSinkExec; use sail_data_source::formats::python::{ InputPartition, PythonDataSourceExec, PythonDataSourceWriteCommitExec, PythonDataSourceWriteExec, @@ -91,14 +104,15 @@ use sail_data_source::formats::socket::{SocketReadOptions, SocketSourceExec}; use sail_data_source::formats::text::source::TextSource; use sail_data_source::formats::text::writer::{TextSink, TextWriterOptions}; use sail_data_source::listing::delete::FileDeleteExec; -use sail_data_source::options::gen::RateReadOptions; +use sail_data_source::options::r#gen::RateReadOptions; use sail_delta_lake::physical_plan::{ DeletionVectorRowsWriterExec, DeletionVectorWriterExec, DeltaCommitContext, DeltaCommitExec, DeltaDiscoveryExec, DeltaLogReplayExec, DeltaMetadataStatsExec, DeltaRemoveActionsExec, DeltaScanByAddsExec, DeltaSnapshotContext, DeltaWriteContext, DeltaWriterExec, - RelaxedTzCastExec, }; -use sail_delta_lake::spec::{Action, ColumnMappingMode, DeltaOperation, StructType}; +use sail_delta_lake::spec::{ + Action, ColumnMappingMode, ColumnMetadataKey, DeltaOperation, StructType, +}; use sail_function::aggregate::bitmap_and_agg::BitmapAndAggFunction; use sail_function::aggregate::bitmap_construct_agg::BitmapConstructAggFunction; use sail_function::aggregate::bitmap_or_agg::BitmapOrAggFunction; @@ -130,12 +144,14 @@ use sail_function::scalar::array::spark_sequence::SparkSequence; use sail_function::scalar::array_struct_field::ArrayStructField; use sail_function::scalar::collection::spark_concat::SparkConcat; use sail_function::scalar::collection::spark_reverse::SparkReverse; +use sail_function::scalar::csv::SparkSchemaOfCsv; use sail_function::scalar::csv::spark_from_csv::SparkFromCSV; use sail_function::scalar::csv::spark_to_csv::SparkToCsv; -use sail_function::scalar::csv::SparkSchemaOfCsv; use sail_function::scalar::datetime::convert_tz::ConvertTz; use sail_function::scalar::datetime::negate_duration::NegateDuration; use sail_function::scalar::datetime::spark_date::SparkDate; +use sail_function::scalar::datetime::spark_date_format::SparkDateFormat; +use sail_function::scalar::datetime::spark_date_part::SparkDatePart; use sail_function::scalar::datetime::spark_date_trunc::SparkDateTrunc; use sail_function::scalar::datetime::spark_interval::{ SparkCalendarInterval, SparkDayTimeInterval, SparkYearMonthInterval, @@ -149,14 +165,13 @@ use sail_function::scalar::datetime::spark_time::SparkTime; use sail_function::scalar::datetime::spark_time_diff::SparkTimeDiff; use sail_function::scalar::datetime::spark_time_trunc::SparkTimeTrunc; use sail_function::scalar::datetime::spark_timestamp::SparkTimestamp; -use sail_function::scalar::datetime::spark_to_chrono_fmt::SparkToChronoFmt; use sail_function::scalar::datetime::spark_try_to_timestamp::SparkTryToTimestamp; use sail_function::scalar::datetime::spark_unix_timestamp::SparkUnixTimestamp; use sail_function::scalar::datetime::spark_window_buckets::SparkWindowBuckets; use sail_function::scalar::datetime::spark_year::SparkYear; use sail_function::scalar::datetime::timestamp_now::TimestampNow; use sail_function::scalar::drop_struct_field::DropStructField; -use sail_function::scalar::explode::{explode_name_to_kind, Explode}; +use sail_function::scalar::explode::{Explode, explode_name_to_kind}; use sail_function::scalar::geo::st_asbinary::StAsBinary; use sail_function::scalar::geo::st_geogfromwkb::StGeogFromWKB; use sail_function::scalar::geo::st_geomfromwkb::StGeomFromWKB; @@ -205,6 +220,7 @@ use sail_function::scalar::string::soundex::Soundex; use sail_function::scalar::string::spark_base64::{SparkBase64, SparkUnbase64}; use sail_function::scalar::string::spark_concat_ws::SparkConcatWs; use sail_function::scalar::string::spark_encode_decode::{SparkDecode, SparkEncode}; +use sail_function::scalar::string::spark_length::{SparkBitLength, SparkOctetLength}; use sail_function::scalar::string::spark_mask::SparkMask; use sail_function::scalar::string::spark_quote::SparkQuote; use sail_function::scalar::string::spark_regexp_extract_all::{ @@ -227,24 +243,30 @@ use sail_function::scalar::variant::spark_to_variant_object::SparkToVariantObjec use sail_function::scalar::variant::spark_variant_explode::SparkVariantExplodeUdf; use sail_function::scalar::variant::spark_variant_get::SparkVariantGet; use sail_function::scalar::variant::spark_variant_to_json::SparkVariantToJsonUdf; +use sail_function::scalar::xml::from_xml::SparkFromXml; use sail_function::scalar::xml::to_xml::SparkToXml; use sail_function::scalar::xml::xpath::Xpath; -use sail_function::scalar::xml::xpath_typed::{xpath_typed_name_to_kind, XpathTyped}; -use sail_function::window::SparkNtile; +use sail_function::scalar::xml::xpath_typed::{XpathTyped, xpath_typed_name_to_kind}; +use sail_function::window::{SparkFirstLastValue, SparkFirstLastValueKind, SparkNtile}; use sail_iceberg::physical_plan::{ - IcebergCommitExec, IcebergDeleteApplyExec, IcebergDiscoveryExec, IcebergManifestScanExec, + IcebergCommitExec, IcebergDeleteApplyExec, IcebergDiscoveryExec, + IcebergEqualityDeleteWriterExec, IcebergManifestScanExec, IcebergMergeMetadataExec, IcebergScanByDataFilesExec, IcebergWriterExec, }; -use sail_iceberg::IcebergWriterExecOptions; +use sail_iceberg::{IcebergWriterExecOptions, SnapshotUpdateKind}; use sail_logical_plan::range::Range; use sail_logical_plan::show_string::{ShowStringFormat, ShowStringStyle}; use sail_physical_plan::barrier::BarrierExec; use sail_physical_plan::catalog_command::CatalogCommandExec; use sail_physical_plan::coalesce::CoalesceExec; +use sail_physical_plan::data_source::RemoteDataSourceExec; use sail_physical_plan::map_partitions::MapPartitionsExec; use sail_physical_plan::merge_cardinality_check::MergeCardinalityCheckExec; use sail_physical_plan::monotonic_id::MonotonicIdExec; use sail_physical_plan::range::RangeExec; +use sail_physical_plan::remote_checkpoint::{ + CheckpointDataSource, RemoteCheckpointCommitExec, RemoteCheckpointWriteExec, +}; use sail_physical_plan::schema_pivot::SchemaPivotExec; use sail_physical_plan::show_string::ShowStringExec; use sail_physical_plan::spark_partition_id::SparkPartitionIdExec; @@ -260,22 +282,22 @@ use sail_python_udf::udf::pyspark_map_iter_udf::{PySparkMapIterKind, PySparkMapI use sail_python_udf::udf::pyspark_udaf::{PySparkGroupAggKind, PySparkGroupAggregateUDF}; use sail_python_udf::udf::pyspark_udf::{PySparkUDF, PySparkUdfKind}; use sail_python_udf::udf::pyspark_udtf::{PySparkUDTF, PySparkUdtfKind}; -use serde::de::DeserializeOwned; use serde::Serialize; +use serde::de::DeserializeOwned; use url::Url; -use crate::plan::gen::extended_aggregate_udf::UdafKind; -use crate::plan::gen::extended_physical_expr_node::ExprKind; -use crate::plan::gen::extended_physical_plan_node::NodeKind; -use crate::plan::gen::extended_scalar_udf::UdfKind; -use crate::plan::gen::extended_stream_udf::StreamUdfKind; -use crate::plan::gen::extended_window_udf::UdwfKind; -use crate::plan::gen::{ +use crate::plan::r#gen::extended_aggregate_udf::UdafKind; +use crate::plan::r#gen::extended_physical_expr_node::ExprKind; +use crate::plan::r#gen::extended_physical_plan_node::NodeKind; +use crate::plan::r#gen::extended_scalar_udf::UdfKind; +use crate::plan::r#gen::extended_stream_udf::StreamUdfKind; +use crate::plan::r#gen::extended_window_udf::UdwfKind; +use crate::plan::r#gen::{ CastColumnExprNode, ExtendedAggregateUdf, ExtendedPhysicalExprNode, ExtendedPhysicalPlanNode, ExtendedScalarUdf, ExtendedStreamUdf, ExtendedWindowUdf, LambdaExprNode, LambdaVariableExprNode, }; -use crate::plan::{gen, StageInputExec}; +use crate::plan::{StageInputExec, r#gen}; use crate::proto::converter::RemotePhysicalProtoConverter; use crate::proto::decode::{ try_decode_field_ref, try_decode_message, try_decode_physical_expr, try_decode_physical_plan, @@ -309,7 +331,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { None => return plan_err!("no physical plan node found"), }; match node_kind { - NodeKind::Range(gen::RangeExecNode { + NodeKind::Range(r#gen::RangeExecNode { start, end, step, @@ -326,7 +348,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { projection, )?)) } - NodeKind::ShowString(gen::ShowStringExecNode { + NodeKind::ShowString(r#gen::ShowStringExecNode { input, names, limit, @@ -346,7 +368,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { Arc::new(schema), ))) } - NodeKind::StageInput(gen::StageInputExecNode { + NodeKind::StageInput(r#gen::StageInputExecNode { input, eq_properties, partitioning, @@ -374,7 +396,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let node = StageInputExec::new(input as usize, properties); Ok(Arc::new(node)) } - NodeKind::SystemTable(gen::SystemTableExecNode { + NodeKind::SystemTable(r#gen::SystemTableExecNode { table, projection, filters, @@ -393,7 +415,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let node = SystemTableExec::try_new(table, projection, filters, fetch)?; Ok(Arc::new(node)) } - NodeKind::SchemaPivot(gen::SchemaPivotExecNode { + NodeKind::SchemaPivot(r#gen::SchemaPivotExecNode { input, names, schema, @@ -405,7 +427,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { Arc::new(schema), ))) } - NodeKind::MapPartitions(gen::MapPartitionsExecNode { input, udf, schema }) => { + NodeKind::MapPartitions(r#gen::MapPartitionsExecNode { input, udf, schema }) => { let Some(udf) = udf else { return plan_err!("no UDF found for MapPartitionsExec"); }; @@ -416,13 +438,14 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { Arc::new(schema), ))) } - NodeKind::Memory(gen::MemoryExecNode { + NodeKind::Memory(r#gen::MemoryExecNode { partitions, schema, projection, show_sizes, sort_information, limit, + output_partitioning, }) => { let schema = try_decode_schema(&schema)?; let partitions = partitions @@ -438,24 +461,84 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .with_show_sizes(show_sizes) .try_with_sort_information(sort_information)? .with_limit(limit.map(|x| x as usize)); - Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) + let scan = DataSourceExec::new(Arc::new(source)); + let scan = if output_partitioning.is_empty() { + scan + } else { + let output_partitioning = + self.try_decode_partitioning(&output_partitioning, &scan.schema(), ctx)?; + scan.with_partitioning(output_partitioning) + }; + Ok(Arc::new(scan)) } - NodeKind::Values(gen::ValuesExecNode { data, schema }) => { + NodeKind::RemoteCheckpointWrite(r#gen::RemoteCheckpointWriteExecNode { + input, + object_store_url, + prefix, + storage_schema, + }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + let object_store_url = + datafusion::execution::object_store::ObjectStoreUrl::parse(object_store_url)?; + let prefix = object_store::path::Path::parse(prefix) + .map_err(|error| plan_datafusion_err!("invalid checkpoint prefix: {error}"))?; + let storage_schema = Arc::new(try_decode_schema(&storage_schema)?); + Ok(Arc::new(RemoteCheckpointWriteExec::try_new( + input, + object_store_url, + prefix, + storage_schema, + )?)) + } + NodeKind::RemoteCheckpointCommit(r#gen::RemoteCheckpointCommitExecNode { + input, + relation_id, + object_store_url, + prefix, + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + let object_store_url = + datafusion::execution::object_store::ObjectStoreUrl::parse(object_store_url)?; + let prefix = object_store::path::Path::parse(prefix) + .map_err(|error| plan_datafusion_err!("invalid checkpoint prefix: {error}"))?; + let logical_schema = Arc::new(try_decode_schema(&logical_schema)?); + let storage_schema = Arc::new(try_decode_schema(&storage_schema)?); + let output_partitioning = + self.try_decode_partitioning(&output_partitioning, &storage_schema, ctx)?; + let output_ordering = output_ordering + .map(|ordering| self.try_decode_lex_ordering(&ordering, &storage_schema, ctx)) + .transpose()?; + Ok(Arc::new(RemoteCheckpointCommitExec::new( + input, + relation_id, + object_store_url, + prefix, + logical_schema, + storage_schema, + output_partitioning, + output_ordering, + ))) + } + NodeKind::Values(r#gen::ValuesExecNode { data, schema }) => { let schema = try_decode_schema(&schema)?; let data = read_record_batches(&data)?; let source = MemorySourceConfig::try_new_from_batches(Arc::new(schema), data)?; Ok(source) } - NodeKind::NdJson(gen::NdJsonExecNode { + NodeKind::NdJson(r#gen::NdJsonExecNode { base_config, file_compression_type, }) => { let file_compression_type: FileCompressionType = self.try_decode_file_compression_type(file_compression_type)?; - let proto = try_decode_message(&base_config)?; - let table_schema = parse_protobuf_file_scan_schema(&proto)?; + let base_config = try_decode_message(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; let source = parse_protobuf_file_scan_config( - &proto, + &base_config, &PhysicalPlanDecodeContext::new(ctx, self), &RemotePhysicalProtoConverter {}, Arc::new(JsonSource::new(table_schema)), @@ -465,18 +548,108 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .build(); Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) } - NodeKind::Arrow(gen::ArrowExecNode { base_config }) => { - let proto = try_decode_message(&base_config)?; - let table_schema = parse_protobuf_file_scan_schema(&proto)?; + NodeKind::Csv(r#gen::CsvExecNode { + base_config, + options, + }) => { + let base_config = try_decode_message(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; + let options = try_decode_message::(&options)?; + let csv_options: CsvOptions = (&options).try_into()?; + let file_compression_type: FileCompressionType = csv_options.compression.into(); + let source = CsvSource::new(table_schema).with_csv_options(csv_options); let source = parse_protobuf_file_scan_config( - &proto, + &base_config, + &PhysicalPlanDecodeContext::new(ctx, self), + &RemotePhysicalProtoConverter {}, + Arc::new(source), + )?; + let source = FileScanConfigBuilder::from(source) + .with_file_compression_type(file_compression_type) + .build(); + Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) + } + NodeKind::Parquet(r#gen::ParquetExecNode { + base_config, + options, + predicate, + output_partitioning, + struct_field_matching, + timezone_mode, + }) => { + let base_config = try_decode_message(&base_config)?; + let predicate_schema = parse_protobuf_file_scan_schema(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; + let options = + try_decode_message::(&options)?; + let options: TableParquetOptions = (&options).try_into()?; + let predicate = predicate + .map(|predicate| { + try_decode_physical_expr(ctx, self, &predicate, predicate_schema.as_ref()) + }) + .transpose()?; + let object_store_url = match base_config.object_store_url.is_empty() { + false => datafusion::execution::object_store::ObjectStoreUrl::parse( + &base_config.object_store_url, + )?, + true => datafusion::execution::object_store::ObjectStoreUrl::local_filesystem(), + }; + let store = ctx.runtime_env().object_store(object_store_url)?; + let metadata_cache = ctx.runtime_env().cache_manager.get_file_metadata_cache(); + let reader_factory = + Arc::new(CachedParquetFileReaderFactory::new(store, metadata_cache)); + let mut source = ParquetSource::new(table_schema) + .with_parquet_file_reader_factory(reader_factory) + .with_table_parquet_options(options); + if let Some(predicate) = predicate { + source = source.with_predicate(predicate); + } + let source = parse_protobuf_file_scan_config( + &base_config, + &PhysicalPlanDecodeContext::new(ctx, self), + &RemotePhysicalProtoConverter {}, + Arc::new(source), + )?; + let struct_field_matching = + Self::try_decode_struct_field_matching(struct_field_matching)?; + let timezone_mode = Self::try_decode_schema_evolution_timezone_mode(timezone_mode)?; + let adapter_factory = match timezone_mode { + SchemaEvolutionTimezoneMode::Strict => { + SchemaEvolutionPhysicalExprAdapterFactoryWithMatching::new( + struct_field_matching, + ) + } + SchemaEvolutionTimezoneMode::Relaxed => { + SchemaEvolutionPhysicalExprAdapterFactoryWithMatching::new_relaxed_timezone( + struct_field_matching, + ) + } + }; + let source = FileScanConfigBuilder::from(source) + .with_expr_adapter(Some(Arc::new(adapter_factory))) + .build(); + let scan = DataSourceExec::new(Arc::new(source)); + let scan = if output_partitioning.is_empty() { + scan + } else { + let output_partitioning = + self.try_decode_partitioning(&output_partitioning, &scan.schema(), ctx)?; + scan.with_partitioning(output_partitioning) + }; + Ok(Arc::new(scan)) + } + NodeKind::Arrow(r#gen::ArrowExecNode { base_config }) => { + let base_config = try_decode_message(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; + let source = parse_protobuf_file_scan_config( + &base_config, &PhysicalPlanDecodeContext::new(ctx, self), &RemotePhysicalProtoConverter {}, Arc::new(ArrowSource::new_file_source(table_schema)), )?; Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) } - NodeKind::Text(gen::TextExecNode { + NodeKind::Text(r#gen::TextExecNode { base_config, file_compression_type, whole_text, @@ -494,10 +667,10 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { ); } }; - let proto = try_decode_message(&base_config)?; - let table_schema = parse_protobuf_file_scan_schema(&proto)?; + let base_config = try_decode_message(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; let source = parse_protobuf_file_scan_config( - &proto, + &base_config, &PhysicalPlanDecodeContext::new(ctx, self), &RemotePhysicalProtoConverter {}, Arc::new(TextSource::new(table_schema, whole_text, line_sep)), @@ -507,52 +680,52 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .build(); Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) } - NodeKind::BinarySource(gen::BinarySourceExecNode { - base_config, - path_glob_filter, - }) => { - let proto = try_decode_message(&base_config)?; - let table_schema = parse_protobuf_file_scan_schema(&proto)?; + NodeKind::BinarySource(r#gen::BinarySourceExecNode { base_config }) => { + let base_config = try_decode_message(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; let source = parse_protobuf_file_scan_config( - &proto, + &base_config, &PhysicalPlanDecodeContext::new(ctx, self), &RemotePhysicalProtoConverter {}, - Arc::new(BinarySource::new(table_schema, path_glob_filter)), + Arc::new(BinarySource::new(table_schema)), )?; let source = FileScanConfigBuilder::from(source).build(); Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) } - NodeKind::Avro(gen::AvroExecNode { base_config }) => { - let proto = try_decode_message(&base_config)?; - let table_schema = parse_protobuf_file_scan_schema(&proto)?; + NodeKind::Avro(r#gen::AvroExecNode { base_config }) => { + let base_config = try_decode_message(&base_config)?; + let table_schema = parse_table_schema_from_proto(&base_config)?; let source = parse_protobuf_file_scan_config( - &proto, + &base_config, &PhysicalPlanDecodeContext::new(ctx, self), &RemotePhysicalProtoConverter {}, Arc::new(AvroSource::new(table_schema)), )?; Ok(Arc::new(DataSourceExec::new(Arc::new(source)))) } - NodeKind::WorkTable(gen::WorkTableExecNode { name, schema }) => { + NodeKind::WorkTable(r#gen::WorkTableExecNode { name, schema }) => { let schema = try_decode_schema(&schema)?; Ok(Arc::new(WorkTableExec::new(name, Arc::new(schema), None)?)) } - NodeKind::RecursiveQuery(gen::RecursiveQueryExecNode { + NodeKind::RecursiveQuery(r#gen::RecursiveQueryExecNode { name, static_term, recursive_term, is_distinct, + output_schema, }) => { let static_term = try_decode_physical_plan(ctx, self, &static_term)?; let recursive_term = try_decode_physical_plan(ctx, self, &recursive_term)?; + let output_schema = Arc::new(try_decode_schema(&output_schema)?); Ok(Arc::new(RecursiveQueryExec::try_new( name, + output_schema, static_term, recursive_term, is_distinct, )?)) } - NodeKind::SortMergeJoin(gen::SortMergeJoinExecNode { + NodeKind::SortMergeJoin(r#gen::SortMergeJoinExecNode { left, right, on, @@ -626,7 +799,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { )?)) } NodeKind::DeltaWriter(delta_writer) => { - let gen::DeltaWriterExecNode { + let r#gen::DeltaWriterExecNode { input, table_url, options, @@ -669,7 +842,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { lakehouse_table, )?)) } - NodeKind::DeltaCommit(gen::DeltaCommitExecNode { + NodeKind::DeltaCommit(r#gen::DeltaCommitExecNode { input, table_url, partition_columns, @@ -710,7 +883,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { lakehouse_table, ))) } - NodeKind::DeltaScanByAdds(gen::DeltaScanByAddsExecNode { + NodeKind::DeltaScanByAdds(r#gen::DeltaScanByAddsExecNode { input, table_url, table_schema, @@ -722,6 +895,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { version, statistics, lakehouse_table_json, + catalog_managed_commits_json, }) => { let input = try_decode_physical_plan(ctx, self, &input)?; let table_url = Url::parse(&table_url) @@ -762,6 +936,14 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map(|bytes| self.try_decode_statistics(bytes)) .transpose()?; let lakehouse_table = self.try_decode_lakehouse_table(&lakehouse_table_json)?; + let catalog_managed_commits = if catalog_managed_commits_json.is_empty() { + None + } else { + Some(self.try_decode_json( + &catalog_managed_commits_json, + "Delta catalog-managed commit set", + )?) + }; Ok(Arc::new( DeltaScanByAddsExec::new( input, @@ -774,11 +956,12 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { limit, pushdown_filter, lakehouse_table, + catalog_managed_commits, ) .with_output_statistics(statistics), )) } - NodeKind::DeltaDiscovery(gen::DeltaDiscoveryExecNode { + NodeKind::DeltaDiscovery(r#gen::DeltaDiscoveryExecNode { table_url, predicate, table_schema, @@ -814,7 +997,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { input_partition_scan, )?)) } - NodeKind::DeltaMetadataStats(gen::DeltaMetadataStatsExecNode { + NodeKind::DeltaMetadataStats(r#gen::DeltaMetadataStatsExecNode { input, stats_schema, }) => { @@ -822,7 +1005,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let stats_schema = Arc::new(try_decode_schema(&stats_schema)?); Ok(Arc::new(DeltaMetadataStatsExec::new(input, stats_schema))) } - NodeKind::DeltaRemoveActions(gen::DeltaRemoveActionsExecNode { + NodeKind::DeltaRemoveActions(r#gen::DeltaRemoveActionsExecNode { input, partition_value_columns_json, }) => { @@ -837,7 +1020,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { partition_value_columns, )?)) } - NodeKind::DeltaLogReplay(gen::DeltaLogReplayExecNode { + NodeKind::DeltaLogReplay(r#gen::DeltaLogReplayExecNode { input, table_url, version, @@ -851,7 +1034,8 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|e| plan_datafusion_err!("failed to parse table URL: {e}"))?; match (checkpoint_input.as_ref(), commits_input.as_ref()) { (Some(checkpoint_input), Some(commits_input)) => { - let checkpoint_input = try_decode_physical_plan(ctx, self, checkpoint_input)?; + let checkpoint_input = + try_decode_physical_plan(ctx, self, checkpoint_input)?; let commits_input = try_decode_physical_plan(ctx, self, commits_input)?; Ok(Arc::new(DeltaLogReplayExec::new_hash( checkpoint_input, @@ -879,11 +1063,15 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { ), } } - NodeKind::ConsoleSink(gen::ConsoleSinkExecNode { input }) => { + NodeKind::ConsoleSink(r#gen::ConsoleSinkExecNode { input }) => { let input = try_decode_physical_plan(ctx, self, &input)?; Ok(Arc::new(ConsoleSinkExec::new(input))) } - NodeKind::SocketSource(gen::SocketSourceExecNode { + NodeKind::NoopSink(r#gen::NoopSinkExecNode { input }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + Ok(Arc::new(NoopSinkExec::new(input))) + } + NodeKind::SocketSource(r#gen::SocketSourceExecNode { host, port, max_batch_size, @@ -908,7 +1096,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { projection, )?)) } - NodeKind::RateSource(gen::RateSourceExecNode { + NodeKind::RateSource(r#gen::RateSourceExecNode { rows_per_second, num_partitions, schema, @@ -930,7 +1118,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { projection, )?)) } - NodeKind::TextSink(gen::TextSinkExecNode { + NodeKind::TextSink(r#gen::TextSinkExecNode { input, base_config, schema, @@ -985,7 +1173,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { sort_order, ))) } - NodeKind::FileDelete(gen::FileDeleteExecNode { + NodeKind::FileDelete(r#gen::FileDeleteExecNode { object_store_url, path, }) => { @@ -995,11 +1183,11 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|e| plan_datafusion_err!("invalid file delete path: {e}"))?; Ok(Arc::new(FileDeleteExec::new(object_store_url, path))) } - NodeKind::StreamCollector(gen::StreamCollectorExecNode { input }) => { + NodeKind::StreamCollector(r#gen::StreamCollectorExecNode { input }) => { let input = try_decode_physical_plan(ctx, self, &input)?; Ok(Arc::new(StreamCollectorExec::try_new(input)?)) } - NodeKind::StreamLimit(gen::StreamLimitExecNode { input, skip, fetch }) => { + NodeKind::StreamLimit(r#gen::StreamLimitExecNode { input, skip, fetch }) => { let input = try_decode_physical_plan(ctx, self, &input)?; let skip = usize::try_from(skip) .map_err(|_| plan_datafusion_err!("invalid skip value for StreamLimitExec"))?; @@ -1009,16 +1197,16 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|_| plan_datafusion_err!("invalid fetch value for StreamLimitExec"))?; Ok(Arc::new(StreamLimitExec::try_new(input, skip, fetch)?)) } - NodeKind::StreamFilter(gen::StreamFilterExecNode { input, predicate }) => { + NodeKind::StreamFilter(r#gen::StreamFilterExecNode { input, predicate }) => { let input = try_decode_physical_plan(ctx, self, &input)?; let predicate = try_decode_physical_expr(ctx, self, &predicate, &input.schema())?; Ok(Arc::new(StreamFilterExec::try_new(input, predicate)?)) } - NodeKind::StreamSourceAdapter(gen::StreamSourceAdapterExecNode { input }) => { + NodeKind::StreamSourceAdapter(r#gen::StreamSourceAdapterExecNode { input }) => { let input = try_decode_physical_plan(ctx, self, &input)?; Ok(Arc::new(StreamSourceAdapterExec::new(input))) } - NodeKind::MergeCardinalityCheck(gen::MergeCardinalityCheckExecNode { + NodeKind::MergeCardinalityCheck(r#gen::MergeCardinalityCheckExecNode { input, target_row_id_col, target_present_col, @@ -1029,7 +1217,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { target_present_col, source_present_col, )?)), - NodeKind::MonotonicId(gen::MonotonicIdExecNode { + NodeKind::MonotonicId(r#gen::MonotonicIdExecNode { input, column_name, schema, @@ -1041,7 +1229,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { Arc::new(schema), )?)) } - NodeKind::SparkPartitionId(gen::SparkPartitionIdExecNode { + NodeKind::SparkPartitionId(r#gen::SparkPartitionIdExecNode { input, column_name, schema, @@ -1053,19 +1241,14 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { Arc::new(schema), )?)) } - NodeKind::Coalesce(gen::CoalesceExecNode { + NodeKind::Coalesce(r#gen::CoalesceExecNode { input, output_partitions, }) => Ok(Arc::new(CoalesceExec::new( try_decode_physical_plan(ctx, self, &input)?, usize::try_from(output_partitions).map_err(|e| plan_datafusion_err!("{e}"))?, ))), - NodeKind::RelaxedTzCast(gen::RelaxedTzCastExecNode { input, schema }) => { - let input = try_decode_physical_plan(ctx, self, &input)?; - let schema = Arc::new(try_decode_schema(&schema)?); - Ok(Arc::new(RelaxedTzCastExec::new(input, schema))) - } - NodeKind::DeletionVectorWriter(gen::DeletionVectorWriterExecNode { + NodeKind::DeletionVectorWriter(r#gen::DeletionVectorWriterExecNode { input, table_url, condition, @@ -1102,7 +1285,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { operation, )?)) } - NodeKind::DeletionVectorRowsWriter(gen::DeletionVectorRowsWriterExecNode { + NodeKind::DeletionVectorRowsWriter(r#gen::DeletionVectorRowsWriterExecNode { input, adds_input, table_url, @@ -1140,7 +1323,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { operation, )?)) } - NodeKind::IcebergWriter(gen::IcebergWriterExecNode { + NodeKind::IcebergWriter(r#gen::IcebergWriterExecNode { input, table_url, partition_columns, @@ -1149,6 +1332,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { options, logical_input_schema, lakehouse_table_json, + merge_row_intents, }) => { let input = try_decode_physical_plan(ctx, self, &input)?; let sink_mode = match sink_mode { @@ -1180,33 +1364,52 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { Some(Arc::new(try_decode_schema(&logical_input_schema)?)) }; - Ok(Arc::new(IcebergWriterExec::new( - input, - table_url, - partition_columns, - sink_mode, - table_exists, - options, - logical_input_schema, - ))) + let writer = if merge_row_intents { + IcebergWriterExec::new_merge( + input, + table_url, + partition_columns, + sink_mode, + table_exists, + options, + logical_input_schema, + ) + } else { + IcebergWriterExec::new( + input, + table_url, + partition_columns, + sink_mode, + table_exists, + options, + logical_input_schema, + ) + }; + Ok(Arc::new(writer)) } - NodeKind::IcebergCommit(gen::IcebergCommitExecNode { + NodeKind::IcebergCommit(r#gen::IcebergCommitExecNode { input, table_url, lakehouse_table_json, + validate_read_snapshot, + expected_snapshot_id, + snapshot_update_kind, }) => { let input = try_decode_physical_plan(ctx, self, &input)?; let table_url = Url::parse(&table_url) .map_err(|e| plan_datafusion_err!("failed to parse table URL: {e}"))?; let lakehouse_table = self.try_decode_lakehouse_table(&lakehouse_table_json)?; + let snapshot_update_kind = + Self::try_decode_iceberg_snapshot_update_kind(snapshot_update_kind)?; - Ok(Arc::new(IcebergCommitExec::new( - input, - table_url, - lakehouse_table, - ))) + Ok(Arc::new( + IcebergCommitExec::new(input, table_url, lakehouse_table, snapshot_update_kind) + .with_expected_snapshot_id( + validate_read_snapshot.then_some(expected_snapshot_id), + ), + )) } - NodeKind::IcebergManifestScan(gen::IcebergManifestScanExecNode { + NodeKind::IcebergManifestScan(r#gen::IcebergManifestScanExecNode { table_url, snapshot_json, }) => { @@ -1216,7 +1419,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { })?; Ok(Arc::new(IcebergManifestScanExec::new(table_url, snapshot))) } - NodeKind::IcebergDiscovery(gen::IcebergDiscoveryExecNode { + NodeKind::IcebergDiscovery(r#gen::IcebergDiscoveryExecNode { input, table_url, snapshot_id, @@ -1230,7 +1433,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { input_partition_scan, )?)) } - NodeKind::IcebergScanByDataFiles(gen::IcebergScanByDataFilesExecNode { + NodeKind::IcebergScanByDataFiles(r#gen::IcebergScanByDataFilesExecNode { input, table_url, output_schema, @@ -1243,7 +1446,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { output_schema, ))) } - NodeKind::IcebergDeleteApply(gen::IcebergDeleteApplyExecNode { + NodeKind::IcebergDeleteApply(r#gen::IcebergDeleteApplyExecNode { input, data_file_path, positional_deletes_json, @@ -1273,7 +1476,69 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { iceberg_schema, ))) } - NodeKind::PythonDataSource(gen::PythonDataSourceExecNode { + NodeKind::IcebergMergeMetadata(r#gen::IcebergMergeMetadataExecNode { + input, + data_file_path, + file_column_name, + row_index_column_name, + data_file_partition_spec_id, + data_file_partition_json, + }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + let merge_metadata = if data_file_path.is_empty() { + IcebergMergeMetadataExec::try_new_partitioned_files( + input, + file_column_name.ok_or_else(|| { + plan_datafusion_err!( + "partitioned Iceberg merge metadata plan is missing file column" + ) + })?, + row_index_column_name, + )? + } else { + IcebergMergeMetadataExec::try_new( + input, + data_file_path, + data_file_partition_spec_id.ok_or_else(|| { + plan_datafusion_err!( + "Iceberg merge metadata plan is missing partition spec id" + ) + })?, + data_file_partition_json.ok_or_else(|| { + plan_datafusion_err!( + "Iceberg merge metadata plan is missing partition values" + ) + })?, + file_column_name, + row_index_column_name, + )? + }; + Ok(Arc::new(merge_metadata)) + } + NodeKind::IcebergEqualityDeleteWriter(r#gen::IcebergEqualityDeleteWriterExecNode { + input, + table_url, + table_properties_json, + write_data_path, + write_folder_storage_path, + lakehouse_table_json, + }) => { + let input = try_decode_physical_plan(ctx, self, &input)?; + let table_url = Url::parse(&table_url) + .map_err(|e| plan_datafusion_err!("failed to parse table URL: {e}"))?; + let table_properties = + self.try_decode_json(&table_properties_json, "Iceberg table properties")?; + let lakehouse_table = self.try_decode_lakehouse_table(&lakehouse_table_json)?; + Ok(Arc::new(IcebergEqualityDeleteWriterExec::new( + input, + table_url, + table_properties, + write_data_path, + write_folder_storage_path, + lakehouse_table, + ))) + } + NodeKind::PythonDataSource(r#gen::PythonDataSourceExecNode { pickled_reader, schema, partitions, @@ -1293,7 +1558,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { partitions, ))) } - NodeKind::PythonDataSourceWrite(gen::PythonDataSourceWriteExecNode { + NodeKind::PythonDataSourceWrite(r#gen::PythonDataSourceWriteExecNode { pickled_writer, schema, is_arrow, @@ -1312,7 +1577,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { is_arrow, ))) } - NodeKind::PythonDataSourceWriteCommit(gen::PythonDataSourceWriteCommitExecNode { + NodeKind::PythonDataSourceWriteCommit(r#gen::PythonDataSourceWriteCommitExecNode { pickled_writer, expected_partitions, input, @@ -1324,13 +1589,13 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { expected_partitions as usize, ))) } - NodeKind::CatalogCommand(gen::CatalogCommandExecNode { schema, command }) => { + NodeKind::CatalogCommand(r#gen::CatalogCommandExecNode { schema, command }) => { let schema = Arc::new(try_decode_schema(&schema)?); let command: sail_catalog::command::CatalogCommand = serde_json::from_str(&command) .map_err(|e| plan_datafusion_err!("failed to decode CatalogCommand: {e}"))?; Ok(Arc::new(CatalogCommandExec::new(command, schema))) } - NodeKind::Barrier(gen::BarrierExecNode { + NodeKind::Barrier(r#gen::BarrierExecNode { preconditions, plan, }) => { @@ -1349,7 +1614,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let node_kind = if let Some(range) = node.downcast_ref::() { let schema = try_encode_schema(range.original_schema().as_ref())?; let projection = self.try_encode_projection(range.projection())?; - NodeKind::Range(gen::RangeExecNode { + NodeKind::Range(r#gen::RangeExecNode { start: range.range().start, end: range.range().end, step: range.range().step, @@ -1359,7 +1624,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { }) } else if let Some(show_string) = node.downcast_ref::() { let schema = try_encode_schema(show_string.schema().as_ref())?; - NodeKind::ShowString(gen::ShowStringExecNode { + NodeKind::ShowString(r#gen::ShowStringExecNode { input: try_encode_physical_plan(self, show_string.input().clone())?, names: show_string.names().to_vec(), limit: show_string.limit() as u64, @@ -1379,7 +1644,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { requires_infinite_memory: _, } => false, }; - NodeKind::StageInput(gen::StageInputExecNode { + NodeKind::StageInput(r#gen::StageInputExecNode { input: *stage_input.input() as u64, eq_properties: Some(eq_properties), partitioning, @@ -1388,16 +1653,18 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(system_table) = node.downcast_ref::() { let table = serde_json::to_string(&system_table.table()) .map_err(|e| plan_datafusion_err!("{e}"))?; - let projection = system_table.projection().map(|x| gen::PhysicalProjection { - columns: x.iter().map(|c| *c as u64).collect(), - }); + let projection = system_table + .projection() + .map(|x| r#gen::PhysicalProjection { + columns: x.iter().map(|c| *c as u64).collect(), + }); let filters = system_table .filters() .iter() .map(|expr| try_encode_physical_expr(self, expr)) .collect::>()?; let fetch = system_table.fetch().map(|f| f as u64); - NodeKind::SystemTable(gen::SystemTableExecNode { + NodeKind::SystemTable(r#gen::SystemTableExecNode { table, projection, filters, @@ -1405,7 +1672,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { }) } else if let Some(schema_pivot) = node.downcast_ref::() { let schema = try_encode_schema(schema_pivot.schema().as_ref())?; - NodeKind::SchemaPivot(gen::SchemaPivotExecNode { + NodeKind::SchemaPivot(r#gen::SchemaPivotExecNode { input: try_encode_physical_plan(self, schema_pivot.input().clone())?, names: schema_pivot.names().to_vec(), schema, @@ -1413,15 +1680,37 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(map_partitions) = node.downcast_ref::() { let udf = self.try_encode_stream_udf(map_partitions.udf().as_ref())?; let schema = try_encode_schema(map_partitions.schema().as_ref())?; - NodeKind::MapPartitions(gen::MapPartitionsExecNode { + NodeKind::MapPartitions(r#gen::MapPartitionsExecNode { input: try_encode_physical_plan(self, map_partitions.input().clone())?, udf: Some(udf), schema, }) + } else if let Some(checkpoint) = node.downcast_ref::() { + NodeKind::RemoteCheckpointWrite(r#gen::RemoteCheckpointWriteExecNode { + input: try_encode_physical_plan(self, checkpoint.input().clone())?, + object_store_url: checkpoint.object_store_url().as_str().to_string(), + prefix: checkpoint.prefix().to_string(), + storage_schema: try_encode_schema(checkpoint.storage_schema().as_ref())?, + }) + } else if let Some(checkpoint) = node.downcast_ref::() { + NodeKind::RemoteCheckpointCommit(r#gen::RemoteCheckpointCommitExecNode { + input: try_encode_physical_plan(self, checkpoint.input().clone())?, + relation_id: checkpoint.relation_id().to_string(), + object_store_url: checkpoint.object_store_url().as_str().to_string(), + prefix: checkpoint.prefix().to_string(), + logical_schema: try_encode_schema(checkpoint.logical_schema().as_ref())?, + storage_schema: try_encode_schema(checkpoint.storage_schema().as_ref())?, + output_partitioning: self + .try_encode_partitioning(checkpoint.checkpoint_partitioning())?, + output_ordering: checkpoint + .checkpoint_ordering() + .map(|ordering| self.try_encode_lex_ordering(ordering)) + .transpose()?, + }) } else if let Some(work_table) = node.downcast_ref::() { let name = work_table.name().to_string(); let schema = try_encode_schema(work_table.schema().as_ref())?; - NodeKind::WorkTable(gen::WorkTableExecNode { name, schema }) + NodeKind::WorkTable(r#gen::WorkTableExecNode { name, schema }) } else if let Some(recursive_query) = node.downcast_ref::() { let name = recursive_query.name().to_string(); let static_term = @@ -1429,22 +1718,24 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let recursive_term = try_encode_physical_plan(self, recursive_query.recursive_term().clone())?; let is_distinct = recursive_query.is_distinct(); - NodeKind::RecursiveQuery(gen::RecursiveQueryExecNode { + let output_schema = try_encode_schema(recursive_query.schema().as_ref())?; + NodeKind::RecursiveQuery(r#gen::RecursiveQueryExecNode { name, static_term, recursive_term, is_distinct, + output_schema, }) } else if let Some(sort_merge_join) = node.downcast_ref::() { let left = try_encode_physical_plan(self, sort_merge_join.left().clone())?; let right = try_encode_physical_plan(self, sort_merge_join.right().clone())?; - let on: Vec = sort_merge_join + let on: Vec = sort_merge_join .on() .iter() .map(|(left, right)| { let left = try_encode_physical_expr(self, left)?; let right = try_encode_physical_expr(self, right)?; - Ok(gen::JoinOn { left, right }) + Ok(r#gen::JoinOn { left, right }) }) .collect::>()?; let filter = sort_merge_join @@ -1459,23 +1750,23 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let index = i.index as u32; let side: gen_datafusion_common::JoinSide = i.side.into(); let side = side.as_str_name().to_string(); - gen::ColumnIndex { index, side } + r#gen::ColumnIndex { index, side } }) .collect(); let schema = try_encode_schema(join_filter.schema())?; - Ok(gen::JoinFilter { + Ok(r#gen::JoinFilter { expression, column_indices, schema, }) }) - .map_or(Ok(None), |v: Result| v.map(Some))?; + .map_or(Ok(None), |v: Result| v.map(Some))?; let join_type: ProtoJoinType = sort_merge_join.join_type().into(); let join_type = join_type.as_str_name().to_string(); let sort_options = sort_merge_join .sort_options() .iter() - .map(|x| gen::SortOptions { + .map(|x| r#gen::SortOptions { descending: x.descending, nulls_first: x.nulls_first, }) @@ -1484,7 +1775,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { datafusion::common::NullEquality::NullEqualsNull => true, datafusion::common::NullEquality::NullEqualsNothing => false, }; - NodeKind::SortMergeJoin(gen::SortMergeJoinExecNode { + NodeKind::SortMergeJoin(r#gen::SortMergeJoinExecNode { left, right, on, @@ -1497,13 +1788,22 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let expr = Some(self.try_encode_lex_ordering(partial_sort.expr())?); let input = try_encode_physical_plan(self, partial_sort.input().clone())?; let common_prefix_length = partial_sort.common_prefix_length() as u64; - NodeKind::PartialSort(gen::PartialSortExecNode { + NodeKind::PartialSort(r#gen::PartialSortExecNode { expr, input, common_prefix_length, }) - } else if let Some(data_source) = node.downcast_ref::() { - let source = data_source.data_source(); + } else if let Some(data_source) = node.downcast_ref::() { + let data_source = data_source.data_source(); + let (source, output_partitioning) = + if let Some(checkpoint) = data_source.downcast_ref::() { + ( + checkpoint.source(), + self.try_encode_partitioning(checkpoint.checkpoint_partitioning())?, + ) + } else { + (data_source, Vec::new()) + }; if let Some(file_scan) = source.downcast_ref::() { let file_source = file_scan.file_source(); if let Some(text_source) = file_source.downcast_ref::() { @@ -1514,24 +1814,73 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { )?)?; let file_compression_type = self.try_encode_file_compression_type(file_scan.file_compression_type)?; - NodeKind::Text(gen::TextExecNode { + NodeKind::Text(r#gen::TextExecNode { base_config, file_compression_type, whole_text: text_source.whole_text(), line_sep: text_source.line_sep().map(|x| vec![x]), }) - } else if let Some(binary_source) = file_source.downcast_ref::() { + } else if file_source.is::() { + let base_config = try_encode_message(serialize_file_scan_config( + file_scan, + self, + &RemotePhysicalProtoConverter {}, + )?)?; + NodeKind::BinarySource(r#gen::BinarySourceExecNode { base_config }) + } else if let Some(csv_source) = file_source.downcast_ref::() { + let base_config = try_encode_message(serialize_file_scan_config( + file_scan, + self, + &RemotePhysicalProtoConverter {}, + )?)?; + let csv_options = CsvOptions { + has_header: Some(csv_source.has_header()), + delimiter: csv_source.delimiter(), + quote: csv_source.quote(), + terminator: csv_source.terminator(), + escape: csv_source.escape(), + comment: csv_source.comment(), + newlines_in_values: Some(csv_source.newlines_in_values()), + truncated_rows: Some(csv_source.truncate_rows()), + compression: file_scan.file_compression_type.into(), + ..Default::default() + }; + let options = try_encode_message(gen_datafusion_common::CsvOptions::try_from( + &csv_options, + )?)?; + NodeKind::Csv(r#gen::CsvExecNode { + base_config, + options, + }) + } else if let Some(parquet_source) = file_source.downcast_ref::() { let base_config = try_encode_message(serialize_file_scan_config( file_scan, self, &RemotePhysicalProtoConverter {}, )?)?; - NodeKind::BinarySource(gen::BinarySourceExecNode { + let options = gen_datafusion_common::TableParquetOptions::try_from( + parquet_source.table_parquet_options(), + )?; + let options = try_encode_message(options)?; + let predicate = parquet_source + .filter() + .map(|predicate| try_encode_physical_expr(self, &predicate)) + .transpose()?; + let (struct_field_matching, timezone_mode) = + Self::parquet_schema_evolution_modes(file_scan)?; + NodeKind::Parquet(r#gen::ParquetExecNode { base_config, - path_glob_filter: binary_source.path_glob_filter().cloned(), + options, + predicate, + output_partitioning, + struct_field_matching: Self::try_encode_struct_field_matching( + struct_field_matching, + ), + timezone_mode: Self::try_encode_schema_evolution_timezone_mode( + timezone_mode, + ), }) } else if file_source.is::() { - // TODO: Check if we still need to have JsonSource: https://github.com/apache/datafusion/pull/14224 let base_config = try_encode_message(serialize_file_scan_config( file_scan, self, @@ -1539,30 +1888,28 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { )?)?; let file_compression_type = self.try_encode_file_compression_type(file_scan.file_compression_type)?; - NodeKind::NdJson(gen::NdJsonExecNode { + NodeKind::NdJson(r#gen::NdJsonExecNode { base_config, file_compression_type, }) } else if file_source.is::() { - // TODO: Check if we still need to have ArrowSource: https://github.com/apache/datafusion/pull/14224 let base_config = try_encode_message(serialize_file_scan_config( file_scan, self, &RemotePhysicalProtoConverter {}, )?)?; - NodeKind::Arrow(gen::ArrowExecNode { base_config }) + NodeKind::Arrow(r#gen::ArrowExecNode { base_config }) } else if file_source.is::() { let base_config = try_encode_message(serialize_file_scan_config( file_scan, self, &RemotePhysicalProtoConverter {}, )?)?; - NodeKind::Avro(gen::AvroExecNode { base_config }) + NodeKind::Avro(r#gen::AvroExecNode { base_config }) } else { return plan_err!("unsupported data source node: {data_source:?}"); } } else if let Some(memory) = source.downcast_ref::() { - // TODO: Check if we still need to have MemorySourceConfig: https://github.com/apache/datafusion/pull/14224 // `memory.schema()` is the schema after projection. // We must use the original schema here. let schema = memory.original_schema(); @@ -1574,18 +1921,19 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let projection = memory .projection() .as_ref() - .map(|x| gen::PhysicalProjection { + .map(|x| r#gen::PhysicalProjection { columns: x.iter().map(|c| *c as u64).collect(), }); let schema = try_encode_schema(schema.as_ref())?; let sort_information = self.try_encode_lex_orderings(memory.sort_information())?; - NodeKind::Memory(gen::MemoryExecNode { + NodeKind::Memory(r#gen::MemoryExecNode { partitions, schema, projection, show_sizes: memory.show_sizes(), sort_information, limit: memory.fetch().map(|x| x as u64), + output_partitioning, }) } else { return plan_err!("unsupported data source node: {data_source:?}"); @@ -1593,7 +1941,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(delta_writer_exec) = node.downcast_ref::() { let input = try_encode_physical_plan(self, delta_writer_exec.input().clone())?; let sink_mode = self.try_encode_physical_sink_mode(delta_writer_exec.sink_mode())?; - NodeKind::DeltaWriter(Box::new(gen::DeltaWriterExecNode { + NodeKind::DeltaWriter(Box::new(r#gen::DeltaWriterExecNode { input, table_url: delta_writer_exec.table_url().to_string(), options: serde_json::to_string(delta_writer_exec.options()) @@ -1612,7 +1960,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(delta_commit_exec) = node.downcast_ref::() { let input = try_encode_physical_plan(self, delta_commit_exec.input().clone())?; let sink_mode = self.try_encode_physical_sink_mode(delta_commit_exec.sink_mode())?; - NodeKind::DeltaCommit(gen::DeltaCommitExecNode { + NodeKind::DeltaCommit(r#gen::DeltaCommitExecNode { input, table_url: delta_commit_exec.table_url().to_string(), partition_columns: delta_commit_exec.partition_columns().to_vec(), @@ -1636,7 +1984,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .projection() .map(|p| { self.try_encode_projection(p) - .map(|columns| gen::PhysicalProjection { columns }) + .map(|columns| r#gen::PhysicalProjection { columns }) }) .transpose() .map_err(|_| plan_datafusion_err!("invalid projection for DeltaScanByAddsExec"))?; @@ -1652,7 +2000,11 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { }; let statistics = Some(self.try_encode_statistics(delta_scan_by_adds_exec.statistics())?); - NodeKind::DeltaScanByAdds(gen::DeltaScanByAddsExecNode { + let catalog_managed_commits_json = delta_scan_by_adds_exec + .catalog_managed_commits() + .map(|value| self.try_encode_json(value, "Delta catalog-managed commit set")) + .unwrap_or_else(|| Ok(String::new()))?; + NodeKind::DeltaScanByAdds(r#gen::DeltaScanByAddsExecNode { input, table_url: delta_scan_by_adds_exec.table_url().to_string(), table_schema, @@ -1665,6 +2017,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { statistics, lakehouse_table_json: self .try_encode_lakehouse_table(delta_scan_by_adds_exec.lakehouse_table())?, + catalog_managed_commits_json, }) } else if let Some(delta_discovery_exec) = node.downcast_ref::() { let input = Some(try_encode_physical_plan( @@ -1681,7 +2034,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else { None }; - NodeKind::DeltaDiscovery(gen::DeltaDiscoveryExecNode { + NodeKind::DeltaDiscovery(r#gen::DeltaDiscoveryExecNode { table_url: delta_discovery_exec.table_url().to_string(), predicate, table_schema, @@ -1693,7 +2046,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(delta_metadata_stats_exec) = node.downcast_ref::() { - NodeKind::DeltaMetadataStats(gen::DeltaMetadataStatsExecNode { + NodeKind::DeltaMetadataStats(r#gen::DeltaMetadataStatsExecNode { input: try_encode_physical_plan(self, delta_metadata_stats_exec.input().clone())?, stats_schema: try_encode_schema(delta_metadata_stats_exec.stats_schema())?, }) @@ -1707,14 +2060,18 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map(serde_json::to_string) .transpose() .map_err(|e| plan_datafusion_err!("{e}"))?; - NodeKind::DeltaRemoveActions(gen::DeltaRemoveActionsExecNode { + NodeKind::DeltaRemoveActions(r#gen::DeltaRemoveActionsExecNode { input, partition_value_columns_json, }) } else if let Some(delta_log_replay_exec) = node.downcast_ref::() { let children = delta_log_replay_exec.children(); let (input, checkpoint_input, commits_input) = match children.as_slice() { - [input] => (try_encode_physical_plan(self, (*input).clone())?, None, None), + [input] => ( + try_encode_physical_plan(self, (*input).clone())?, + None, + None, + ), [checkpoint_input, commits_input] => ( Vec::new(), Some(try_encode_physical_plan(self, (*checkpoint_input).clone())?), @@ -1723,10 +2080,10 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { _ => { return plan_err!( "DeltaLogReplayExec expects one child for sort replay or two children for hash replay" - ) + ); } }; - NodeKind::DeltaLogReplay(gen::DeltaLogReplayExecNode { + NodeKind::DeltaLogReplay(r#gen::DeltaLogReplayExecNode { input, table_url: delta_log_replay_exec.table_url().to_string(), version: delta_log_replay_exec.version(), @@ -1738,7 +2095,10 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { }) } else if let Some(console_sink) = node.downcast_ref::() { let input = try_encode_physical_plan(self, console_sink.input().clone())?; - NodeKind::ConsoleSink(gen::ConsoleSinkExecNode { input }) + NodeKind::ConsoleSink(r#gen::ConsoleSinkExecNode { input }) + } else if let Some(noop_sink) = node.downcast_ref::() { + let input = try_encode_physical_plan(self, noop_sink.input().clone())?; + NodeKind::NoopSink(r#gen::NoopSinkExecNode { input }) } else if let Some(socket_source) = node.downcast_ref::() { let options = socket_source.options(); let max_batch_size = u64::try_from(options.max_batch_size).map_err(|_| { @@ -1746,7 +2106,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { })?; let schema = try_encode_schema(socket_source.original_schema())?; let projection = self.try_encode_projection(socket_source.projection())?; - NodeKind::SocketSource(gen::SocketSourceExecNode { + NodeKind::SocketSource(r#gen::SocketSourceExecNode { host: options.host.clone(), port: options.port as u32, max_batch_size, @@ -1764,7 +2124,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { })?; let schema = try_encode_schema(rate_source.original_schema())?; let projection = self.try_encode_projection(rate_source.projection())?; - NodeKind::RateSource(gen::RateSourceExecNode { + NodeKind::RateSource(r#gen::RateSourceExecNode { rows_per_second, num_partitions, schema, @@ -1798,7 +2158,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .into_iter() .map(try_encode_message) .collect::>()?; - Some(gen::PhysicalSortExprNodeCollection { + Some(r#gen::PhysicalSortExprNodeCollection { physical_sort_expr_nodes, }) } else { @@ -1814,7 +2174,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { self.try_encode_compression_type_variant(writer_options.compression)?; let line_sep = vec![writer_options.line_sep]; let schema = try_encode_schema(data_sink.schema().as_ref())?; - NodeKind::TextSink(gen::TextSinkExecNode { + NodeKind::TextSink(r#gen::TextSinkExecNode { input, base_config, schema, @@ -1827,7 +2187,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } } else if let Some(stream_collector) = node.downcast_ref::() { let input = try_encode_physical_plan(self, stream_collector.input().clone())?; - NodeKind::StreamCollector(gen::StreamCollectorExecNode { input }) + NodeKind::StreamCollector(r#gen::StreamCollectorExecNode { input }) } else if let Some(stream_limit) = node.downcast_ref::() { let input = try_encode_physical_plan(self, stream_limit.input().clone())?; let skip = u64::try_from(stream_limit.skip()).map_err(|_| { @@ -1840,17 +2200,17 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|_| { plan_datafusion_err!("cannot encode fetch value for StreamLimitExec") })?; - NodeKind::StreamLimit(gen::StreamLimitExecNode { input, skip, fetch }) + NodeKind::StreamLimit(r#gen::StreamLimitExecNode { input, skip, fetch }) } else if let Some(stream_filter) = node.downcast_ref::() { let input = try_encode_physical_plan(self, stream_filter.input().clone())?; let predicate = try_encode_physical_expr(self, stream_filter.predicate())?; - NodeKind::StreamFilter(gen::StreamFilterExecNode { input, predicate }) + NodeKind::StreamFilter(r#gen::StreamFilterExecNode { input, predicate }) } else if let Some(stream_source_adapter) = node.downcast_ref::() { let input = try_encode_physical_plan(self, stream_source_adapter.input().clone())?; - NodeKind::StreamSourceAdapter(gen::StreamSourceAdapterExecNode { input }) + NodeKind::StreamSourceAdapter(r#gen::StreamSourceAdapterExecNode { input }) } else if let Some(cardinality_check) = node.downcast_ref::() { let input = try_encode_physical_plan(self, cardinality_check.input().clone())?; - NodeKind::MergeCardinalityCheck(gen::MergeCardinalityCheckExecNode { + NodeKind::MergeCardinalityCheck(r#gen::MergeCardinalityCheckExecNode { input, target_row_id_col: cardinality_check.target_row_id_col().to_string(), target_present_col: cardinality_check.target_present_col().to_string(), @@ -1859,7 +2219,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(monotonic_id) = node.downcast_ref::() { let input = try_encode_physical_plan(self, monotonic_id.input().clone())?; let schema = try_encode_schema(monotonic_id.schema().as_ref())?; - NodeKind::MonotonicId(gen::MonotonicIdExecNode { + NodeKind::MonotonicId(r#gen::MonotonicIdExecNode { input, column_name: monotonic_id.column_name().to_string(), schema, @@ -1867,22 +2227,18 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(spark_partition_id) = node.downcast_ref::() { let input = try_encode_physical_plan(self, spark_partition_id.input().clone())?; let schema = try_encode_schema(spark_partition_id.schema().as_ref())?; - NodeKind::SparkPartitionId(gen::SparkPartitionIdExecNode { + NodeKind::SparkPartitionId(r#gen::SparkPartitionIdExecNode { input, column_name: spark_partition_id.column_name().to_string(), schema, }) } else if let Some(coalesce) = node.downcast_ref::() { let input = try_encode_physical_plan(self, coalesce.input().clone())?; - NodeKind::Coalesce(gen::CoalesceExecNode { + NodeKind::Coalesce(r#gen::CoalesceExecNode { input, output_partitions: u64::try_from(coalesce.output_partitions()) .map_err(|e| plan_datafusion_err!("{e}"))?, }) - } else if let Some(relaxed_tz_cast) = node.downcast_ref::() { - let input = try_encode_physical_plan(self, relaxed_tz_cast.input().clone())?; - let schema = try_encode_schema(relaxed_tz_cast.schema().as_ref())?; - NodeKind::RelaxedTzCast(gen::RelaxedTzCastExecNode { input, schema }) } else if let Some(dv_writer_exec) = node.downcast_ref::() { let input = try_encode_physical_plan(self, dv_writer_exec.input().clone())?; let condition = try_encode_physical_expr(self, dv_writer_exec.condition())?; @@ -1892,7 +2248,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else { None }; - NodeKind::DeletionVectorWriter(gen::DeletionVectorWriterExecNode { + NodeKind::DeletionVectorWriter(r#gen::DeletionVectorWriterExecNode { input, table_url: dv_writer_exec.table_url().to_string(), condition, @@ -1916,7 +2272,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else { None }; - NodeKind::DeletionVectorRowsWriter(gen::DeletionVectorRowsWriterExecNode { + NodeKind::DeletionVectorRowsWriter(r#gen::DeletionVectorRowsWriterExecNode { input, adds_input, table_url: dv_rows_writer_exec.table_url().to_string(), @@ -1940,7 +2296,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map(|schema| try_encode_schema(schema.as_ref())) .transpose()? .unwrap_or_default(); - NodeKind::IcebergWriter(gen::IcebergWriterExecNode { + NodeKind::IcebergWriter(r#gen::IcebergWriterExecNode { input, table_url: iceberg_writer_exec.table_url().to_string(), partition_columns: iceberg_writer_exec @@ -1954,25 +2310,32 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { logical_input_schema, lakehouse_table_json: self .try_encode_lakehouse_table(iceberg_writer_exec.lakehouse_table())?, + merge_row_intents: iceberg_writer_exec.reads_merge_row_intents(), }) } else if let Some(iceberg_commit_exec) = node.downcast_ref::() { let input = try_encode_physical_plan(self, iceberg_commit_exec.input().clone())?; - NodeKind::IcebergCommit(gen::IcebergCommitExecNode { + let expected_snapshot_id = iceberg_commit_exec.expected_snapshot_id(); + NodeKind::IcebergCommit(r#gen::IcebergCommitExecNode { input, table_url: iceberg_commit_exec.table_url().to_string(), lakehouse_table_json: self .try_encode_lakehouse_table(iceberg_commit_exec.lakehouse_table())?, + validate_read_snapshot: expected_snapshot_id.is_some(), + expected_snapshot_id: expected_snapshot_id.flatten(), + snapshot_update_kind: Self::try_encode_iceberg_snapshot_update_kind( + iceberg_commit_exec.snapshot_update_kind(), + ), }) } else if let Some(manifest_scan) = node.downcast_ref::() { let snapshot_json = serde_json::to_string(manifest_scan.snapshot()) .map_err(|e| plan_datafusion_err!("failed to encode Iceberg snapshot: {e}"))?; - NodeKind::IcebergManifestScan(gen::IcebergManifestScanExecNode { + NodeKind::IcebergManifestScan(r#gen::IcebergManifestScanExecNode { table_url: manifest_scan.table_url().to_string(), snapshot_json, }) } else if let Some(discovery) = node.downcast_ref::() { let input = try_encode_physical_plan(self, discovery.input().clone())?; - NodeKind::IcebergDiscovery(gen::IcebergDiscoveryExecNode { + NodeKind::IcebergDiscovery(r#gen::IcebergDiscoveryExecNode { input, table_url: discovery.table_url().to_string(), snapshot_id: discovery.snapshot_id(), @@ -1981,7 +2344,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(scan_by_files) = node.downcast_ref::() { let input = try_encode_physical_plan(self, scan_by_files.input().clone())?; let output_schema = try_encode_schema(scan_by_files.output_schema().as_ref())?; - NodeKind::IcebergScanByDataFiles(gen::IcebergScanByDataFilesExecNode { + NodeKind::IcebergScanByDataFiles(r#gen::IcebergScanByDataFilesExecNode { input, table_url: scan_by_files.table_url().to_string(), output_schema, @@ -1996,7 +2359,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|e| plan_datafusion_err!("failed to encode equality delete refs: {e}"))?; let iceberg_schema_json = serde_json::to_string(delete_apply.iceberg_schema()) .map_err(|e| plan_datafusion_err!("failed to encode Iceberg schema: {e}"))?; - NodeKind::IcebergDeleteApply(gen::IcebergDeleteApplyExecNode { + NodeKind::IcebergDeleteApply(r#gen::IcebergDeleteApplyExecNode { input, data_file_path: delete_apply.data_file_path().to_string(), positional_deletes_json, @@ -2004,17 +2367,52 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { table_url: delete_apply.table_url().to_string(), iceberg_schema_json, }) + } else if let Some(merge_metadata) = node.downcast_ref::() { + let input = try_encode_physical_plan(self, merge_metadata.input().clone())?; + NodeKind::IcebergMergeMetadata(r#gen::IcebergMergeMetadataExecNode { + input, + data_file_path: merge_metadata + .data_file_path() + .unwrap_or_default() + .to_string(), + file_column_name: merge_metadata.file_column_name().map(ToString::to_string), + row_index_column_name: merge_metadata + .row_index_column_name() + .map(ToString::to_string), + data_file_partition_spec_id: merge_metadata.data_file_partition_spec_id(), + data_file_partition_json: merge_metadata + .data_file_partition_json() + .map(ToString::to_string), + }) + } else if let Some(equality_writer) = node.downcast_ref::() + { + let input = try_encode_physical_plan(self, equality_writer.input().clone())?; + let table_properties_json = self.try_encode_json( + equality_writer.table_properties(), + "Iceberg table properties", + )?; + NodeKind::IcebergEqualityDeleteWriter(r#gen::IcebergEqualityDeleteWriterExecNode { + input, + table_url: equality_writer.table_url().to_string(), + table_properties_json, + write_data_path: equality_writer.write_data_path().map(ToString::to_string), + write_folder_storage_path: equality_writer + .write_folder_storage_path() + .map(ToString::to_string), + lakehouse_table_json: self + .try_encode_lakehouse_table(equality_writer.lakehouse_table())?, + }) } else if let Some(python_exec) = node.downcast_ref::() { let schema = try_encode_schema(python_exec.schema().as_ref())?; let partitions = python_exec .partitions() .iter() - .map(|p| gen::PythonDataSourceInputPartition { + .map(|p| r#gen::PythonDataSourceInputPartition { partition_id: p.partition_id as u64, data: p.data.clone(), }) .collect(); - NodeKind::PythonDataSource(gen::PythonDataSourceExecNode { + NodeKind::PythonDataSource(r#gen::PythonDataSourceExecNode { pickled_reader: python_exec.pickled_reader().to_vec(), schema, partitions, @@ -2022,7 +2420,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { } else if let Some(python_write_exec) = node.downcast_ref::() { let schema = try_encode_schema(python_write_exec.input().schema().as_ref())?; let input = try_encode_physical_plan(self, python_write_exec.input().clone())?; - NodeKind::PythonDataSourceWrite(gen::PythonDataSourceWriteExecNode { + NodeKind::PythonDataSourceWrite(r#gen::PythonDataSourceWriteExecNode { pickled_writer: python_write_exec.pickled_writer().to_vec(), schema, is_arrow: python_write_exec.is_arrow(), @@ -2032,7 +2430,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { node.downcast_ref::() { let input = try_encode_physical_plan(self, python_commit_exec.input().clone())?; - NodeKind::PythonDataSourceWriteCommit(gen::PythonDataSourceWriteCommitExecNode { + NodeKind::PythonDataSourceWriteCommit(r#gen::PythonDataSourceWriteCommitExecNode { pickled_writer: python_commit_exec.pickled_writer().to_vec(), expected_partitions: python_commit_exec.expected_partitions() as u64, input, @@ -2041,9 +2439,9 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let schema = try_encode_schema(catalog_command_exec.schema().as_ref())?; let command = serde_json::to_string(catalog_command_exec.command()) .map_err(|e| plan_datafusion_err!("failed to encode CatalogCommand: {e}"))?; - NodeKind::CatalogCommand(gen::CatalogCommandExecNode { schema, command }) + NodeKind::CatalogCommand(r#gen::CatalogCommandExecNode { schema, command }) } else if let Some(file_delete_exec) = node.downcast_ref::() { - NodeKind::FileDelete(gen::FileDeleteExecNode { + NodeKind::FileDelete(r#gen::FileDeleteExecNode { object_store_url: file_delete_exec.object_store_url().as_str().to_string(), path: file_delete_exec.path().to_string(), }) @@ -2054,7 +2452,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map(|child| try_encode_physical_plan(self, child.clone())) .collect::>()?; let plan = try_encode_physical_plan(self, barrier_exec.plan().clone())?; - NodeKind::Barrier(gen::BarrierExecNode { + NodeKind::Barrier(r#gen::BarrierExecNode { preconditions, plan, }) @@ -2098,8 +2496,8 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { None => return plan_err!("ExtendedScalarUdf: no UDF found for {name}"), }; match udf_kind { - UdfKind::Standard(gen::StandardUdf {}) => {} - UdfKind::PySpark(gen::PySparkUdf { + UdfKind::Standard(r#gen::StandardUdf {}) => {} + UdfKind::PySpark(r#gen::PySparkUdf { kind, name, payload, @@ -2129,7 +2527,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { ); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::PySparkCoGroupMap(gen::PySparkCoGroupMapUdf { + UdfKind::PySparkCoGroupMap(r#gen::PySparkCoGroupMapUdf { name, payload, deterministic, @@ -2168,79 +2566,105 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { )?; return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::DropStructField(gen::DropStructFieldUdf { field_names }) => { + UdfKind::DropStructField(r#gen::DropStructFieldUdf { field_names }) => { let udf = DropStructField::new(field_names); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::Explode(gen::ExplodeUdf { name }) => { + UdfKind::Explode(r#gen::ExplodeUdf { name }) => { let kind = explode_name_to_kind(&name)?; let udf = Explode::new(kind); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::XpathTyped(gen::XpathTypedUdf { name }) => { + UdfKind::XpathTyped(r#gen::XpathTypedUdf { name }) => { let kind = xpath_typed_name_to_kind(&name)?; let udf = XpathTyped::new(kind); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkToXml(gen::SparkToXmlUdf { session_timezone }) => { + UdfKind::SparkToXml(r#gen::SparkToXmlUdf { session_timezone }) => { let udf = SparkToXml::new(Arc::from(session_timezone)); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkUnixTimestamp(gen::SparkUnixTimestampUdf { timezone }) => { - let udf = SparkUnixTimestamp::new(Arc::from(timezone)); + UdfKind::SparkFromXml(r#gen::SparkFromXmlUdf { session_timezone }) => { + let udf = SparkFromXml::new(Arc::from(session_timezone)); + return Ok(Arc::new(ScalarUDF::from(udf))); + } + UdfKind::SparkUnixTimestamp(r#gen::SparkUnixTimestampUdf { + session_timezone, + ansi_mode, + }) => { + let udf = SparkUnixTimestamp::new(Arc::from(session_timezone), ansi_mode); + return Ok(Arc::new(ScalarUDF::from(udf))); + } + UdfKind::SparkDateFormat(r#gen::SparkDateFormatUdf { session_timezone }) => { + let udf = SparkDateFormat::new(Arc::from(session_timezone)); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::StructFunction(gen::StructFunctionUdf { field_names }) => { + UdfKind::StructFunction(r#gen::StructFunctionUdf { field_names }) => { let udf = StructFunction::new(field_names); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::ArraysZip(gen::ArraysZipUdf { field_names }) => { + UdfKind::ArraysZip(r#gen::ArraysZipUdf { field_names }) => { let udf = ArraysZip::new(field_names); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::UpdateStructField(gen::UpdateStructFieldUdf { field_names }) => { + UdfKind::UpdateStructField(r#gen::UpdateStructFieldUdf { field_names }) => { let udf = UpdateStructField::new(field_names); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::TimestampNow(gen::TimestampNowUdf { - timezone, + UdfKind::TimestampNow(r#gen::TimestampNowUdf { + session_timezone, time_unit, }) => { let time_unit = gen_datafusion_common::TimeUnit::from_str_name(time_unit.as_str()) .ok_or_else(|| plan_datafusion_err!("invalid time unit: {time_unit}"))?; let time_unit: TimeUnit = time_unit.into(); - let udf = TimestampNow::new(Arc::from(timezone), time_unit); + let udf = TimestampNow::new(Arc::from(session_timezone), time_unit); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkTimestamp(gen::SparkTimestampUdf { timezone, is_try }) => { - let udf = SparkTimestamp::try_new(timezone.map(Arc::from), is_try)?; + UdfKind::SparkTimestamp(r#gen::SparkTimestampUdf { + timezone, + is_try, + ansi_mode, + }) => { + let udf = SparkTimestamp::try_new(timezone.map(Arc::from), ansi_mode, is_try)?; return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkDate(gen::SparkDateUdf { is_try }) => { + UdfKind::SparkDate(r#gen::SparkDateUdf { is_try }) => { return Ok(Arc::new(ScalarUDF::from(SparkDate::new(is_try)))); } - UdfKind::SparkTime(gen::SparkTimeUdf { is_try }) => { + UdfKind::SparkTryToTimestamp(r#gen::SparkTryToTimestampUdf { timezone }) => { + return Ok(Arc::new(ScalarUDF::from(SparkTryToTimestamp::try_new( + timezone.map(Arc::from), + )))); + } + UdfKind::SparkTime(r#gen::SparkTimeUdf { is_try }) => { return Ok(Arc::new(ScalarUDF::from(SparkTime::new(is_try)))); } - UdfKind::SparkFromCsv(gen::SparkFromCsvUdf { session_timezone }) => { + UdfKind::SparkCeil(r#gen::SparkCeilUdf { ansi_mode }) => { + return Ok(Arc::new(ScalarUDF::from(SparkCeil::new(ansi_mode)))); + } + UdfKind::SparkFloor(r#gen::SparkFloorUdf { ansi_mode }) => { + return Ok(Arc::new(ScalarUDF::from(SparkFloor::new(ansi_mode)))); + } + UdfKind::SparkFromCsv(r#gen::SparkFromCsvUdf { session_timezone }) => { let udf = SparkFromCSV::new(Arc::from(session_timezone)); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkToCsv(gen::SparkToCsvUdf { session_timezone }) => { + UdfKind::SparkToCsv(r#gen::SparkToCsvUdf { session_timezone }) => { let udf = SparkToCsv::new(Arc::from(session_timezone)); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkFromJson(gen::SparkFromJsonUdf { session_timezone }) => { + UdfKind::SparkFromJson(r#gen::SparkFromJsonUdf { session_timezone }) => { let udf = SparkFromJson::new(Arc::from(session_timezone)); return Ok(Arc::new(ScalarUDF::from(udf))); } - UdfKind::SparkVariantGet(gen::SparkVariantGetUdf { safe }) => { + UdfKind::SparkVariantGet(r#gen::SparkVariantGetUdf { safe }) => { return Ok(Arc::new(ScalarUDF::from(SparkVariantGet::new(safe)))); } - UdfKind::SparkNextDay(gen::SparkNextDayUdf { ansi_mode }) => { + UdfKind::SparkNextDay(r#gen::SparkNextDayUdf { ansi_mode }) => { return Ok(Arc::new(ScalarUDF::from(SparkNextDay::new(ansi_mode)))); } - UdfKind::SparkWindowBuckets(gen::SparkWindowBucketsUdf { + UdfKind::SparkWindowBuckets(r#gen::SparkWindowBucketsUdf { window_duration, slide_duration, start_time, @@ -2251,36 +2675,36 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { start_time, )))); } - UdfKind::SparkToNumber(gen::SparkToNumberUdf { safe }) => { + UdfKind::SparkToNumber(r#gen::SparkToNumberUdf { safe }) => { return Ok(Arc::new(ScalarUDF::from(SparkToNumber::new(safe)))); } - UdfKind::SparkToChar(gen::SparkToCharUdf { ansi_mode }) => { + UdfKind::SparkToChar(r#gen::SparkToCharUdf { ansi_mode }) => { return Ok(Arc::new(ScalarUDF::from(SparkToChar::new(ansi_mode)))); } - UdfKind::SparkAbs(gen::SparkAbsUdf { ansi_mode }) => { + UdfKind::SparkAbs(r#gen::SparkAbsUdf { ansi_mode }) => { return Ok(Arc::new(ScalarUDF::from(SparkAbs::new(ansi_mode)))); } - UdfKind::SparkBin(gen::SparkBinUdf { ansi_mode }) => { + UdfKind::SparkBin(r#gen::SparkBinUdf { ansi_mode }) => { return Ok(Arc::new(ScalarUDF::from(SparkBin::new(ansi_mode)))); } - UdfKind::SparkPmod(gen::SparkPmodUdf { ansi_mode }) => { + UdfKind::SparkPmod(r#gen::SparkPmodUdf { ansi_mode }) => { return Ok(Arc::new(ScalarUDF::from(SparkPmod::new(ansi_mode)))); } - UdfKind::SparkNegative(gen::SparkNegativeUdf { ansi_mode }) => { + UdfKind::SparkNegative(r#gen::SparkNegativeUdf { ansi_mode }) => { return Ok(Arc::new(ScalarUDF::from(SparkNegative::new(ansi_mode)))); } - UdfKind::SparkMakeTimestampNtz(gen::SparkMakeTimestampNtzUdf { is_try }) => { + UdfKind::SparkMakeTimestampNtz(r#gen::SparkMakeTimestampNtzUdf { is_try }) => { return Ok(Arc::new(ScalarUDF::from(SparkMakeTimestampNtz::new( is_try, )))); } - UdfKind::ConvertTz(gen::ConvertTzUdf { classic }) => { + UdfKind::ConvertTz(r#gen::ConvertTzUdf { classic }) => { return Ok(Arc::new(ScalarUDF::from(ConvertTz::new(classic)))); } - UdfKind::SparkParseJson(gen::SparkParseJsonUdf { safe }) => { + UdfKind::SparkParseJson(r#gen::SparkParseJsonUdf { safe }) => { return Ok(Arc::new(ScalarUDF::from(SparkParseJson::new(safe)))); } - UdfKind::SparkStructRename(gen::SparkStructRenameUdf { target_type }) => { + UdfKind::SparkStructRename(r#gen::SparkStructRenameUdf { target_type }) => { let target_type = self.try_decode_data_type(&target_type)?; return Ok(Arc::new(ScalarUDF::from(SparkStructRename::new( target_type, @@ -2305,8 +2729,11 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { "format_string" => Ok(Arc::new(ScalarUDF::from(FormatStringFunc::new()))), "greatest" => Ok(Arc::new(ScalarUDF::from(GreatestFunc::new()))), "least" => Ok(Arc::new(ScalarUDF::from(LeastFunc::new()))), + "length" => Ok(Arc::new(ScalarUDF::from(SparkLengthFunc::new()))), "levenshtein" => Ok(Arc::new(ScalarUDF::from(Levenshtein::new()))), "make_valid_utf8" => Ok(Arc::new(ScalarUDF::from(MakeValidUtf8::new()))), + "spark_bit_length" => Ok(Arc::new(ScalarUDF::from(SparkBitLength::new()))), + "spark_octet_length" => Ok(Arc::new(ScalarUDF::from(SparkOctetLength::new()))), "map_entries" => Ok(Arc::new(ScalarUDF::from(SparkMapEntries::new()))), "map_from_arrays" => Ok(Arc::new(ScalarUDF::from(MapFromArrays::new()))), "map_from_entries" => Ok(Arc::new(ScalarUDF::from(MapFromEntries::new()))), @@ -2397,6 +2824,9 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { "spark_conv" | "conv" => Ok(Arc::new(ScalarUDF::from(SparkConv::new()))), "spark_signum" | "signum" => Ok(Arc::new(ScalarUDF::from(SparkSignum::new()))), "spark_last_day" | "last_day" => Ok(Arc::new(ScalarUDF::from(SparkLastDay::new()))), + "spark_date_part" | "date_part" | "datepart" | "extract" => { + Ok(Arc::new(ScalarUDF::from(SparkDatePart::new()))) + } "spark_year" | "year" => Ok(Arc::new(ScalarUDF::from(SparkYear::new()))), "spark_luhn_check" | "luhn_check" => { Ok(Arc::new(ScalarUDF::from(SparkLuhnCheck::new()))) @@ -2431,13 +2861,16 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { "spark_calendar_interval" => { Ok(Arc::new(ScalarUDF::from(SparkCalendarInterval::new()))) } - "spark_to_chrono_fmt" => Ok(Arc::new(ScalarUDF::from(SparkToChronoFmt::new()))), + "spark_date_format" | "date_format" => { + // Use UTC as default timezone when creating from name only + Ok(Arc::new(ScalarUDF::from(SparkDateFormat::new(Arc::from( + "UTC", + ))))) + } "spark_try_to_timestamp" | "try_to_timestamp" => { Ok(Arc::new(ScalarUDF::from(SparkTryToTimestamp::new()))) } "spark_expm1" | "expm1" => Ok(Arc::new(ScalarUDF::from(SparkExpm1::new()))), - "spark_ceil" | "ceil" => Ok(Arc::new(ScalarUDF::from(SparkCeil::new()))), - "spark_floor" | "floor" => Ok(Arc::new(ScalarUDF::from(SparkFloor::new()))), "spark_to_utf8" => Ok(Arc::new(ScalarUDF::from(SparkToUtf8::new()))), "spark_to_large_utf8" => Ok(Arc::new(ScalarUDF::from(SparkToLargeUtf8::new()))), "spark_to_utf8_view" => Ok(Arc::new(ScalarUDF::from(SparkToUtf8View::new()))), @@ -2491,6 +2924,9 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node_inner.is::() || node_inner.is::() || node_inner.is::() + || node_inner.is::() + || node_inner.is::() + || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() @@ -2507,23 +2943,21 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node_inner.is::() || node_inner.is::() || node_inner.is::() - || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() - || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() + || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() - || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() @@ -2555,7 +2989,6 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node_inner.is::() || node_inner.is::() || node_inner.is::() - || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() @@ -2568,7 +3001,6 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node_inner.is::() || node_inner.is::() || node_inner.is::() - || node_inner.is::() || node_inner.is::() || node_inner.is::() || node_inner.is::() @@ -2589,11 +3021,12 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node_inner.is::() || node_inner.is::() || node_inner.is::() + || matches!(node.name(), "date_part" | "datepart" | "extract") || node.name() == "json_as_text" || node.name() == "json_len" || node.name() == "json_length" { - UdfKind::Standard(gen::StandardUdf {}) + UdfKind::Standard(r#gen::StandardUdf {}) } else if let Some(func) = node.inner().downcast_ref::() { let kind = self.try_encode_pyspark_udf_kind(func.kind())?; let input_types = func @@ -2603,7 +3036,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .collect::>>()?; let output_type = self.try_encode_data_type(func.output_type())?; let config = self.try_encode_pyspark_udf_config(func.config())?; - UdfKind::PySpark(gen::PySparkUdf { + UdfKind::PySpark(r#gen::PySparkUdf { kind, name: func.name().to_string(), payload: func.payload().to_vec(), @@ -2625,7 +3058,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .collect::>>()?; let output_type = self.try_encode_data_type(func.output_type())?; let config = self.try_encode_pyspark_udf_config(func.config())?; - UdfKind::PySparkCoGroupMap(gen::PySparkCoGroupMapUdf { + UdfKind::PySparkCoGroupMap(r#gen::PySparkCoGroupMapUdf { name: func.name().to_string(), payload: func.payload().to_vec(), deterministic: func.deterministic(), @@ -2639,97 +3072,121 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { }) } else if let Some(func) = node.inner().downcast_ref::() { let field_names = func.field_names().to_vec(); - UdfKind::DropStructField(gen::DropStructFieldUdf { field_names }) + UdfKind::DropStructField(r#gen::DropStructFieldUdf { field_names }) } else if let Some(_func) = node.inner().downcast_ref::() { let name = node.name().to_string(); - UdfKind::Explode(gen::ExplodeUdf { name }) + UdfKind::Explode(r#gen::ExplodeUdf { name }) } else if let Some(_func) = node.inner().downcast_ref::() { let name = node.name().to_string(); - UdfKind::XpathTyped(gen::XpathTypedUdf { name }) + UdfKind::XpathTyped(r#gen::XpathTypedUdf { name }) } else if let Some(func) = node.inner().downcast_ref::() { let session_timezone = func.session_timezone().to_string(); - UdfKind::SparkToXml(gen::SparkToXmlUdf { session_timezone }) + UdfKind::SparkToXml(r#gen::SparkToXmlUdf { session_timezone }) + } else if let Some(func) = node.inner().downcast_ref::() { + let session_timezone = func.session_timezone().to_string(); + UdfKind::SparkFromXml(r#gen::SparkFromXmlUdf { session_timezone }) } else if let Some(func) = node.inner().downcast_ref::() { - let timezone = func.timezone().to_string(); - UdfKind::SparkUnixTimestamp(gen::SparkUnixTimestampUdf { timezone }) + let session_timezone = func.session_timezone().to_string(); + let ansi_mode = func.ansi_mode(); + UdfKind::SparkUnixTimestamp(r#gen::SparkUnixTimestampUdf { + session_timezone, + ansi_mode, + }) + } else if let Some(func) = node.inner().downcast_ref::() { + let session_timezone = func.session_timezone().to_string(); + UdfKind::SparkDateFormat(r#gen::SparkDateFormatUdf { session_timezone }) } else if let Some(func) = node.inner().downcast_ref::() { let field_names = func.field_names().to_vec(); - UdfKind::StructFunction(gen::StructFunctionUdf { field_names }) + UdfKind::StructFunction(r#gen::StructFunctionUdf { field_names }) } else if let Some(func) = node.inner().downcast_ref::() { let field_names = func.field_names().to_vec(); - UdfKind::ArraysZip(gen::ArraysZipUdf { field_names }) + UdfKind::ArraysZip(r#gen::ArraysZipUdf { field_names }) } else if let Some(func) = node.inner().downcast_ref::() { let field_names = func.field_names().to_vec(); - UdfKind::UpdateStructField(gen::UpdateStructFieldUdf { field_names }) + UdfKind::UpdateStructField(r#gen::UpdateStructFieldUdf { field_names }) } else if let Some(func) = node.inner().downcast_ref::() { - let timezone = func.timezone().to_string(); + let session_timezone = func.session_timezone().to_string(); let time_unit: gen_datafusion_common::TimeUnit = func.time_unit().into(); let time_unit = time_unit.as_str_name().to_string(); - UdfKind::TimestampNow(gen::TimestampNowUdf { - timezone, + UdfKind::TimestampNow(r#gen::TimestampNowUdf { + session_timezone, time_unit, }) } else if let Some(func) = node.inner().downcast_ref::() { let timezone = func.timezone().map(|x| x.to_string()); let is_try = func.is_try(); - UdfKind::SparkTimestamp(gen::SparkTimestampUdf { timezone, is_try }) + let ansi_mode = func.ansi_mode(); + UdfKind::SparkTimestamp(r#gen::SparkTimestampUdf { + timezone, + is_try, + ansi_mode, + }) } else if let Some(func) = node.inner().downcast_ref::() { let is_try = func.is_try(); - UdfKind::SparkDate(gen::SparkDateUdf { is_try }) + UdfKind::SparkDate(r#gen::SparkDateUdf { is_try }) + } else if let Some(func) = node.inner().downcast_ref::() { + let timezone = func.timezone().map(|x| x.to_string()); + UdfKind::SparkTryToTimestamp(r#gen::SparkTryToTimestampUdf { timezone }) } else if let Some(func) = node.inner().downcast_ref::() { let is_try = func.is_try(); - UdfKind::SparkTime(gen::SparkTimeUdf { is_try }) + UdfKind::SparkTime(r#gen::SparkTimeUdf { is_try }) } else if let Some(func) = node.inner().downcast_ref::() { let safe = func.safe(); - UdfKind::SparkVariantGet(gen::SparkVariantGetUdf { safe }) + UdfKind::SparkVariantGet(r#gen::SparkVariantGetUdf { safe }) } else if let Some(func) = node.inner().downcast_ref::() { let safe = func.safe(); - UdfKind::SparkParseJson(gen::SparkParseJsonUdf { safe }) + UdfKind::SparkParseJson(r#gen::SparkParseJsonUdf { safe }) } else if let Some(func) = node.inner().downcast_ref::() { let session_timezone = func.session_timezone().to_string(); - UdfKind::SparkFromCsv(gen::SparkFromCsvUdf { session_timezone }) + UdfKind::SparkFromCsv(r#gen::SparkFromCsvUdf { session_timezone }) } else if let Some(func) = node.inner().downcast_ref::() { let session_timezone = func.session_timezone().to_string(); - UdfKind::SparkToCsv(gen::SparkToCsvUdf { session_timezone }) + UdfKind::SparkToCsv(r#gen::SparkToCsvUdf { session_timezone }) } else if let Some(func) = node.inner().downcast_ref::() { let session_timezone = func.session_timezone().to_string(); - UdfKind::SparkFromJson(gen::SparkFromJsonUdf { session_timezone }) + UdfKind::SparkFromJson(r#gen::SparkFromJsonUdf { session_timezone }) } else if let Some(func) = node.inner().downcast_ref::() { let ansi_mode = func.ansi_mode(); - UdfKind::SparkNextDay(gen::SparkNextDayUdf { ansi_mode }) + UdfKind::SparkNextDay(r#gen::SparkNextDayUdf { ansi_mode }) } else if let Some(func) = node.inner().downcast_ref::() { - UdfKind::SparkWindowBuckets(gen::SparkWindowBucketsUdf { + UdfKind::SparkWindowBuckets(r#gen::SparkWindowBucketsUdf { window_duration: func.window_duration(), slide_duration: func.slide_duration(), start_time: func.start_time(), }) } else if let Some(func) = node.inner().downcast_ref::() { let safe = func.safe(); - UdfKind::SparkToNumber(gen::SparkToNumberUdf { safe }) + UdfKind::SparkToNumber(r#gen::SparkToNumberUdf { safe }) } else if let Some(func) = node.inner().downcast_ref::() { let ansi_mode = func.ansi_mode(); - UdfKind::SparkToChar(gen::SparkToCharUdf { ansi_mode }) + UdfKind::SparkToChar(r#gen::SparkToCharUdf { ansi_mode }) } else if let Some(func) = node.inner().downcast_ref::() { let ansi_mode = func.ansi_mode(); - UdfKind::SparkAbs(gen::SparkAbsUdf { ansi_mode }) + UdfKind::SparkAbs(r#gen::SparkAbsUdf { ansi_mode }) } else if let Some(func) = node.inner().downcast_ref::() { let ansi_mode = func.ansi_mode(); - UdfKind::SparkBin(gen::SparkBinUdf { ansi_mode }) + UdfKind::SparkBin(r#gen::SparkBinUdf { ansi_mode }) } else if let Some(func) = node.inner().downcast_ref::() { let ansi_mode = func.ansi_mode(); - UdfKind::SparkPmod(gen::SparkPmodUdf { ansi_mode }) + UdfKind::SparkPmod(r#gen::SparkPmodUdf { ansi_mode }) } else if let Some(func) = node.inner().downcast_ref::() { let ansi_mode = func.ansi_mode(); - UdfKind::SparkNegative(gen::SparkNegativeUdf { ansi_mode }) + UdfKind::SparkNegative(r#gen::SparkNegativeUdf { ansi_mode }) + } else if let Some(func) = node.inner().downcast_ref::() { + let ansi_mode = func.ansi_mode(); + UdfKind::SparkCeil(r#gen::SparkCeilUdf { ansi_mode }) + } else if let Some(func) = node.inner().downcast_ref::() { + let ansi_mode = func.ansi_mode(); + UdfKind::SparkFloor(r#gen::SparkFloorUdf { ansi_mode }) } else if let Some(func) = node.inner().downcast_ref::() { let is_try = func.is_try(); - UdfKind::SparkMakeTimestampNtz(gen::SparkMakeTimestampNtzUdf { is_try }) + UdfKind::SparkMakeTimestampNtz(r#gen::SparkMakeTimestampNtzUdf { is_try }) } else if let Some(func) = node.inner().downcast_ref::() { let classic = func.classic(); - UdfKind::ConvertTz(gen::ConvertTzUdf { classic }) + UdfKind::ConvertTz(r#gen::ConvertTzUdf { classic }) } else if let Some(func) = node.inner().downcast_ref::() { let target_type = self.try_encode_data_type(func.target_type())?; - UdfKind::SparkStructRename(gen::SparkStructRenameUdf { target_type }) + UdfKind::SparkStructRename(r#gen::SparkStructRenameUdf { target_type }) } else { return Ok(()); }; @@ -2745,7 +3202,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|e| plan_datafusion_err!("failed to decode udaf: {e}"))?; let ExtendedAggregateUdf { udaf_kind } = udaf; match udaf_kind { - Some(UdafKind::Standard(gen::StandardUdaf {})) => match name { + Some(UdafKind::Standard(r#gen::StandardUdaf {})) => match name { "bitmap_and_agg" => Ok(Arc::new(AggregateUDF::from(BitmapAndAggFunction::new()))), "bitmap_construct_agg" => Ok(Arc::new(AggregateUDF::from( BitmapConstructAggFunction::new(), @@ -2765,7 +3222,6 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { "min_by" => Ok(Arc::new(AggregateUDF::from(MinByFunction::new()))), "mode" => Ok(Arc::new(AggregateUDF::from(ModeFunction::new()))), "percentile" => Ok(Arc::new(AggregateUDF::from(PercentileFunction::new()))), - "percentile_disc" => Ok(Arc::new(AggregateUDF::from(PercentileDisc::new()))), "product" => Ok(Arc::new(AggregateUDF::from(ProductFunction::new()))), "regr_avgx" => Ok(Arc::new(AggregateUDF::from(Regr::new( RegrType::AvgX, @@ -2818,7 +3274,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { "try_sum" => Ok(Arc::new(AggregateUDF::from(SparkTrySum::new()))), _ => plan_err!("Could not find Aggregate Function: {name}"), }, - Some(UdafKind::PySparkGroupAgg(gen::PySparkGroupAggUdaf { + Some(UdafKind::PySparkGroupAgg(r#gen::PySparkGroupAggUdaf { name, payload, deterministic, @@ -2855,7 +3311,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { ); Ok(Arc::new(AggregateUDF::from(udaf))) } - Some(UdafKind::PySparkGroupMap(gen::PySparkGroupMapUdaf { + Some(UdafKind::PySparkGroupMap(r#gen::PySparkGroupMapUdaf { name, payload, deterministic, @@ -2887,7 +3343,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { ); Ok(Arc::new(AggregateUDF::from(udaf))) } - Some(UdafKind::PySparkBatchCollector(gen::PySparkBatchCollectorUdaf { + Some(UdafKind::PySparkBatchCollector(r#gen::PySparkBatchCollectorUdaf { input_types, input_names, })) => { @@ -2898,6 +3354,9 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let udaf = PySparkBatchCollectorUDF::new(input_types, input_names); Ok(Arc::new(AggregateUDF::from(udaf))) } + Some(UdafKind::PercentileDisc(r#gen::PercentileDiscUdaf { ansi_mode })) => { + Ok(Arc::new(AggregateUDF::from(PercentileDisc::new(ansi_mode)))) + } None => plan_err!("ExtendedAggregateUdf: no UDF found for {name}"), } } @@ -2916,7 +3375,6 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node.inner().is::() || node.inner().is::() || node.inner().is::() - || node.inner().is::() || node.inner().is::() || node.inner().is::() || node.inner().is::() @@ -2927,7 +3385,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { || node.inner().is::() || node.inner().is::() { - UdafKind::Standard(gen::StandardUdaf {}) + UdafKind::Standard(r#gen::StandardUdaf {}) } else if let Some(func) = node.inner().downcast_ref::() { let input_types = func .input_types() @@ -2937,7 +3395,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let output_type = self.try_encode_data_type(func.output_type())?; let config = self.try_encode_pyspark_udf_config(func.config())?; let kind = self.try_encode_pyspark_group_agg_kind(func.kind())?; - UdafKind::PySparkGroupAgg(gen::PySparkGroupAggUdaf { + UdafKind::PySparkGroupAgg(r#gen::PySparkGroupAggUdaf { name: func.name().to_string(), payload: func.payload().to_vec(), deterministic: func.deterministic(), @@ -2956,7 +3414,7 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .collect::>>()?; let output_type = self.try_encode_data_type(func.output_type())?; let config = self.try_encode_pyspark_udf_config(func.config())?; - UdafKind::PySparkGroupMap(gen::PySparkGroupMapUdaf { + UdafKind::PySparkGroupMap(r#gen::PySparkGroupMapUdaf { name: func.name().to_string(), payload: func.payload().to_vec(), deterministic: func.deterministic(), @@ -2973,10 +3431,14 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .iter() .map(|x| self.try_encode_data_type(x)) .collect::>>()?; - UdafKind::PySparkBatchCollector(gen::PySparkBatchCollectorUdaf { + UdafKind::PySparkBatchCollector(r#gen::PySparkBatchCollectorUdaf { input_types, input_names: func.input_names().to_vec(), }) + } else if let Some(func) = node.inner().downcast_ref::() { + UdafKind::PercentileDisc(r#gen::PercentileDiscUdaf { + ansi_mode: func.ansi_mode(), + }) } else { return Ok(()); }; @@ -2992,20 +3454,64 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .map_err(|e| plan_datafusion_err!("failed to decode udwf: {e}"))?; let ExtendedWindowUdf { udwf_kind } = udwf; match udwf_kind { - Some(UdwfKind::Standard(gen::StandardUdwf {})) => match name { + Some(UdwfKind::Standard(r#gen::StandardUdwf {})) => match name { + "cume_dist" => Ok(cume_dist_udwf()), + "dense_rank" => Ok(dense_rank_udwf()), + "first" | "first_value" => Ok(first_value_udwf()), + "lag" => Ok(lag_udwf()), + "last" | "last_value" => Ok(last_value_udwf()), + "lead" => Ok(lead_udwf()), + "nth_value" => Ok(nth_value_udwf()), "ntile" => Ok(Arc::new(WindowUDF::from(SparkNtile::new()))), + "rank" => Ok(rank_udwf()), + "row_number" => Ok(row_number_udwf()), + "percent_rank" => Ok(percent_rank_udwf()), _ => plan_err!("Could not find Window Function: {name}"), }, + Some(UdwfKind::SparkFirstLastValue(r#gen::SparkFirstLastValueUdwf { + first, + ignore_nulls, + })) => { + let fun = if first { + SparkFirstLastValue::first(ignore_nulls) + } else { + SparkFirstLastValue::last(ignore_nulls) + }; + Ok(Arc::new(WindowUDF::from(fun))) + } None => plan_err!("ExtendedWindowUdf: no UDWF found for {name}"), } } fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { - if !node.inner().is::() { + let udwf_kind = if node.inner().is::() { + UdwfKind::Standard(r#gen::StandardUdwf {}) + } else if let Some(func) = node.inner().downcast_ref::() { + UdwfKind::SparkFirstLastValue(r#gen::SparkFirstLastValueUdwf { + first: matches!(func.kind(), SparkFirstLastValueKind::First), + ignore_nulls: func.ignore_nulls(), + }) + } else if matches!( + node.name(), + "cume_dist" + | "dense_rank" + | "first" + | "first_value" + | "lag" + | "last" + | "last_value" + | "lead" + | "nth_value" + | "rank" + | "row_number" + | "percent_rank" + ) { + UdwfKind::Standard(r#gen::StandardUdwf {}) + } else { return Ok(()); - } + }; let node = ExtendedWindowUdf { - udwf_kind: Some(UdwfKind::Standard(gen::StandardUdwf {})), + udwf_kind: Some(udwf_kind), }; node.encode(buf) .map_err(|e| plan_datafusion_err!("failed to encode udwf: {e}")) @@ -3023,17 +3529,29 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { .ok_or_else(|| plan_datafusion_err!("missing physical expr node"))?; match expr_kind { ExprKind::SchemaEvolutionCast(node) => { - let (input, input_field, target_field) = self.try_decode_cast_column_expr( - &node, - inputs, - "SchemaEvolutionCastColumnExpr", - )?; - Ok(Arc::new(SchemaEvolutionCastColumnExpr::new( - input, - input_field, - target_field, - None, - ))) + let (input, input_field, target_field, matching, timezone_mode) = self + .try_decode_cast_column_expr(&node, inputs, "SchemaEvolutionCastColumnExpr")?; + let cast = match timezone_mode { + SchemaEvolutionTimezoneMode::Strict => { + SchemaEvolutionCastColumnExpr::new_with_matching( + input, + input_field, + target_field, + None, + matching, + ) + } + SchemaEvolutionTimezoneMode::Relaxed => { + SchemaEvolutionCastColumnExpr::new_relaxed_timezone( + input, + input_field, + target_field, + None, + matching, + ) + } + }; + Ok(Arc::new(cast)) } // Lambdas are handled in converter.rs, but we leave it here for defensive programming. ExprKind::Lambda(node) => { @@ -3065,6 +3583,8 @@ impl PhysicalExtensionCodec for RemoteExecutionCodec { let node = self.try_encode_cast_column_expr( cast.input_field().as_ref(), cast.target_field().as_ref(), + cast.matching(), + cast.timezone_mode(), )?; ExprKind::SchemaEvolutionCast(node) } else if let Some(lambda) = node.downcast_ref::() { @@ -3096,10 +3616,18 @@ impl RemoteExecutionCodec { node: &CastColumnExprNode, inputs: &[Arc], expr_name: &str, - ) -> Result<(Arc, Arc, Arc)> { + ) -> Result<( + Arc, + Arc, + Arc, + StructFieldMatching, + SchemaEvolutionTimezoneMode, + )> { let CastColumnExprNode { input_schema, target_schema, + struct_field_matching, + timezone_mode, } = node; if inputs.len() != 1 { return plan_err!( @@ -3128,6 +3656,8 @@ impl RemoteExecutionCodec { inputs[0].clone(), Arc::new(input_field), Arc::new(target_field), + Self::try_decode_struct_field_matching(*struct_field_matching)?, + Self::try_decode_schema_evolution_timezone_mode(*timezone_mode)?, )) } @@ -3135,6 +3665,8 @@ impl RemoteExecutionCodec { &self, input_field: &Field, target_field: &Field, + matching: StructFieldMatching, + timezone_mode: SchemaEvolutionTimezoneMode, ) -> Result { let input_schema = Schema::new(vec![input_field.clone()]); let input_schema = try_encode_schema(&input_schema)?; @@ -3143,35 +3675,150 @@ impl RemoteExecutionCodec { Ok(CastColumnExprNode { input_schema, target_schema, + struct_field_matching: Self::try_encode_struct_field_matching(matching), + timezone_mode: Self::try_encode_schema_evolution_timezone_mode(timezone_mode), }) } + fn try_decode_struct_field_matching(mode: i32) -> Result { + match r#gen::StructFieldMatching::try_from(mode) + .map_err(|_| plan_datafusion_err!("invalid struct field matching mode"))? + { + r#gen::StructFieldMatching::Name => Ok(StructFieldMatching::Name), + r#gen::StructFieldMatching::PhysicalName => Ok(StructFieldMatching::PhysicalName), + r#gen::StructFieldMatching::FieldId => Ok(StructFieldMatching::FieldId), + } + } + + fn try_encode_struct_field_matching(mode: StructFieldMatching) -> i32 { + (match mode { + StructFieldMatching::Name => r#gen::StructFieldMatching::Name, + StructFieldMatching::PhysicalName => r#gen::StructFieldMatching::PhysicalName, + StructFieldMatching::FieldId => r#gen::StructFieldMatching::FieldId, + }) as i32 + } + + fn try_decode_schema_evolution_timezone_mode(mode: i32) -> Result { + match r#gen::SchemaEvolutionTimezoneMode::try_from(mode) + .map_err(|_| plan_datafusion_err!("invalid schema evolution timezone mode"))? + { + r#gen::SchemaEvolutionTimezoneMode::Strict => Ok(SchemaEvolutionTimezoneMode::Strict), + r#gen::SchemaEvolutionTimezoneMode::Relaxed => Ok(SchemaEvolutionTimezoneMode::Relaxed), + } + } + + fn try_encode_schema_evolution_timezone_mode(mode: SchemaEvolutionTimezoneMode) -> i32 { + (match mode { + SchemaEvolutionTimezoneMode::Strict => r#gen::SchemaEvolutionTimezoneMode::Strict, + SchemaEvolutionTimezoneMode::Relaxed => r#gen::SchemaEvolutionTimezoneMode::Relaxed, + }) as i32 + } + + fn parquet_schema_evolution_modes( + file_scan: &FileScanConfig, + ) -> Result<(StructFieldMatching, SchemaEvolutionTimezoneMode)> { + let Some(factory) = file_scan.expr_adapter_factory.as_ref() else { + return Ok(( + Self::parquet_struct_field_matching(file_scan), + SchemaEvolutionTimezoneMode::Strict, + )); + }; + + let field_name = "__sail_schema_evolution_codec_probe"; + let logical_schema = Arc::new(Schema::new(vec![Field::new( + field_name, + DataType::Int64, + true, + )])); + let physical_schema = Arc::new(Schema::new(vec![Field::new( + field_name, + DataType::Int32, + true, + )])); + let adapter = factory.create(logical_schema, physical_schema)?; + let rewritten = adapter.rewrite(Arc::new(Column::new(field_name, 0)))?; + if let Some(cast) = rewritten.downcast_ref::() { + return Ok((cast.matching(), cast.timezone_mode())); + } + + Ok(( + Self::parquet_struct_field_matching(file_scan), + SchemaEvolutionTimezoneMode::Strict, + )) + } + + fn parquet_struct_field_matching(file_scan: &FileScanConfig) -> StructFieldMatching { + fn field_or_descendant_contains_metadata(field: &Field, keys: &[&str]) -> bool { + if keys.iter().any(|key| field.metadata().contains_key(*key)) { + return true; + } + match field.data_type() { + DataType::Struct(fields) => fields + .iter() + .any(|field| field_or_descendant_contains_metadata(field, keys)), + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) => { + field_or_descendant_contains_metadata(field.as_ref(), keys) + } + _ => false, + } + } + + let fields = file_scan + .file_source() + .table_schema() + .file_schema() + .fields(); + let has_metadata = |keys: &[&str]| { + fields + .iter() + .any(|field| field_or_descendant_contains_metadata(field, keys)) + }; + + // `PARQUET:field_id` is present in both Delta name and ID modes, so prefer + // the mode-specific Delta metadata before using it as a generic fallback. + if has_metadata(&[ + ColumnMetadataKey::ColumnMappingId.as_ref(), + ColumnMetadataKey::ParquetFieldId.as_ref(), + ]) { + StructFieldMatching::FieldId + } else if has_metadata(&[ColumnMetadataKey::ColumnMappingPhysicalName.as_ref()]) { + StructFieldMatching::PhysicalName + } else if has_metadata(&[PARQUET_FIELD_ID_META_KEY]) { + StructFieldMatching::FieldId + } else { + StructFieldMatching::Name + } + } + fn try_decode_physical_sink_mode( &self, - proto_mode: &gen::PhysicalSinkMode, + proto_mode: &r#gen::PhysicalSinkMode, ) -> Result { - let gen::PhysicalSinkMode { mode } = proto_mode; + let r#gen::PhysicalSinkMode { mode } = proto_mode; match mode { - Some(gen::physical_sink_mode::Mode::Append(gen::AppendMode {})) => { + Some(r#gen::physical_sink_mode::Mode::Append(r#gen::AppendMode {})) => { Ok(PhysicalSinkMode::Append) } - Some(gen::physical_sink_mode::Mode::Overwrite(gen::OverwriteMode {})) => { + Some(r#gen::physical_sink_mode::Mode::Overwrite(r#gen::OverwriteMode {})) => { Ok(PhysicalSinkMode::Overwrite) } - Some(gen::physical_sink_mode::Mode::OverwriteIf(gen::OverwriteIfMode { source })) => { - Ok(PhysicalSinkMode::OverwriteIf { - condition: None, - source: source.clone(), - }) - } - Some(gen::physical_sink_mode::Mode::ErrorIfExists(gen::ErrorIfExistsMode {})) => { + Some(r#gen::physical_sink_mode::Mode::OverwriteIf(r#gen::OverwriteIfMode { + source, + })) => Ok(PhysicalSinkMode::OverwriteIf { + condition: None, + source: source.clone(), + }), + Some(r#gen::physical_sink_mode::Mode::ErrorIfExists(r#gen::ErrorIfExistsMode {})) => { Ok(PhysicalSinkMode::ErrorIfExists) } - Some(gen::physical_sink_mode::Mode::IgnoreIfExists(gen::IgnoreIfExistsMode {})) => { + Some(r#gen::physical_sink_mode::Mode::IgnoreIfExists(r#gen::IgnoreIfExistsMode {})) => { Ok(PhysicalSinkMode::IgnoreIfExists) } - Some(gen::physical_sink_mode::Mode::OverwritePartitions( - gen::OverwritePartitionsMode {}, + Some(r#gen::physical_sink_mode::Mode::OverwritePartitions( + r#gen::OverwritePartitionsMode {}, )) => Ok(PhysicalSinkMode::OverwritePartitions), None => plan_err!("PhysicalSinkMode is missing"), } @@ -3180,33 +3827,37 @@ impl RemoteExecutionCodec { fn try_encode_physical_sink_mode( &self, mode: &PhysicalSinkMode, - ) -> Result { + ) -> Result { let mode = match mode { - PhysicalSinkMode::Append => gen::physical_sink_mode::Mode::Append(gen::AppendMode {}), + PhysicalSinkMode::Append => { + r#gen::physical_sink_mode::Mode::Append(r#gen::AppendMode {}) + } PhysicalSinkMode::Overwrite => { - gen::physical_sink_mode::Mode::Overwrite(gen::OverwriteMode {}) + r#gen::physical_sink_mode::Mode::Overwrite(r#gen::OverwriteMode {}) } PhysicalSinkMode::OverwriteIf { source, .. } => { - gen::physical_sink_mode::Mode::OverwriteIf(gen::OverwriteIfMode { + r#gen::physical_sink_mode::Mode::OverwriteIf(r#gen::OverwriteIfMode { source: source.clone(), }) } PhysicalSinkMode::ErrorIfExists => { - gen::physical_sink_mode::Mode::ErrorIfExists(gen::ErrorIfExistsMode {}) + r#gen::physical_sink_mode::Mode::ErrorIfExists(r#gen::ErrorIfExistsMode {}) } PhysicalSinkMode::IgnoreIfExists => { - gen::physical_sink_mode::Mode::IgnoreIfExists(gen::IgnoreIfExistsMode {}) + r#gen::physical_sink_mode::Mode::IgnoreIfExists(r#gen::IgnoreIfExistsMode {}) } PhysicalSinkMode::OverwritePartitions => { - gen::physical_sink_mode::Mode::OverwritePartitions(gen::OverwritePartitionsMode {}) + r#gen::physical_sink_mode::Mode::OverwritePartitions( + r#gen::OverwritePartitionsMode {}, + ) } }; - Ok(gen::PhysicalSinkMode { mode: Some(mode) }) + Ok(r#gen::PhysicalSinkMode { mode: Some(mode) }) } fn try_decode_delta_snapshot_context( &self, - context: &gen::DeltaSnapshotContext, + context: &r#gen::DeltaSnapshotContext, ) -> Result { Ok(DeltaSnapshotContext { version: context.version, @@ -3223,8 +3874,8 @@ impl RemoteExecutionCodec { fn try_encode_delta_snapshot_context( &self, context: &DeltaSnapshotContext, - ) -> Result { - Ok(gen::DeltaSnapshotContext { + ) -> Result { + Ok(r#gen::DeltaSnapshotContext { version: context.version, protocol_json: self.try_encode_json(&context.protocol, "Delta protocol")?, metadata_json: self.try_encode_json(&context.metadata, "Delta metadata")?, @@ -3238,7 +3889,7 @@ impl RemoteExecutionCodec { fn try_decode_delta_commit_context( &self, - context: &gen::DeltaCommitContext, + context: &r#gen::DeltaCommitContext, ) -> Result { let base_snapshot = context .base_snapshot @@ -3251,18 +3902,18 @@ impl RemoteExecutionCodec { fn try_encode_delta_commit_context( &self, context: &DeltaCommitContext, - ) -> Result { + ) -> Result { let base_snapshot = context .base_snapshot .as_ref() .map(|context| self.try_encode_delta_snapshot_context(context)) .transpose()?; - Ok(gen::DeltaCommitContext { base_snapshot }) + Ok(r#gen::DeltaCommitContext { base_snapshot }) } fn try_decode_delta_write_context( &self, - context: &gen::DeltaWriteContext, + context: &r#gen::DeltaWriteContext, ) -> Result { let commit_context = match context.commit_context { Some(ref context) => self.try_decode_delta_commit_context(context)?, @@ -3305,7 +3956,7 @@ impl RemoteExecutionCodec { fn try_encode_delta_write_context( &self, context: &DeltaWriteContext, - ) -> Result { + ) -> Result { let initial_actions_json = context .initial_actions .iter() @@ -3327,7 +3978,7 @@ impl RemoteExecutionCodec { .map(|schema| self.try_encode_json(schema, "Delta logical schema")) .transpose()?; - Ok(gen::DeltaWriteContext { + Ok(r#gen::DeltaWriteContext { commit_context: Some(self.try_encode_delta_commit_context(&context.commit_context)?), final_schema_json: self.try_encode_json(&context.final_schema, "Delta final schema")?, effective_column_mapping_mode: Self::try_encode_delta_column_mapping_mode( @@ -3342,29 +3993,56 @@ impl RemoteExecutionCodec { } fn try_decode_delta_column_mapping_mode(&self, mode: i32) -> Result { - match gen::DeltaColumnMappingMode::try_from(mode) + match r#gen::DeltaColumnMappingMode::try_from(mode) .map_err(|_| plan_datafusion_err!("invalid Delta column mapping mode"))? { - gen::DeltaColumnMappingMode::None => Ok(ColumnMappingMode::None), - gen::DeltaColumnMappingMode::Name => Ok(ColumnMappingMode::Name), - gen::DeltaColumnMappingMode::Id => Ok(ColumnMappingMode::Id), + r#gen::DeltaColumnMappingMode::None => Ok(ColumnMappingMode::None), + r#gen::DeltaColumnMappingMode::Name => Ok(ColumnMappingMode::Name), + r#gen::DeltaColumnMappingMode::Id => Ok(ColumnMappingMode::Id), } } fn try_encode_delta_column_mapping_mode(mode: ColumnMappingMode) -> Result { Ok((match mode { - ColumnMappingMode::None => gen::DeltaColumnMappingMode::None, - ColumnMappingMode::Name => gen::DeltaColumnMappingMode::Name, - ColumnMappingMode::Id => gen::DeltaColumnMappingMode::Id, + ColumnMappingMode::None => r#gen::DeltaColumnMappingMode::None, + ColumnMappingMode::Name => r#gen::DeltaColumnMappingMode::Name, + ColumnMappingMode::Id => r#gen::DeltaColumnMappingMode::Id, }) as i32) } + fn try_decode_iceberg_snapshot_update_kind(kind: i32) -> Result { + match r#gen::IcebergSnapshotUpdateKind::try_from(kind) + .map_err(|_| plan_datafusion_err!("invalid Iceberg snapshot update kind"))? + { + r#gen::IcebergSnapshotUpdateKind::FastAppend => Ok(SnapshotUpdateKind::FastAppend), + r#gen::IcebergSnapshotUpdateKind::FullOverwrite => { + Ok(SnapshotUpdateKind::FullOverwrite) + } + r#gen::IcebergSnapshotUpdateKind::RowDelta => Ok(SnapshotUpdateKind::RowDelta), + r#gen::IcebergSnapshotUpdateKind::Unspecified => { + plan_err!("Iceberg snapshot update kind is unspecified") + } + } + } + + fn try_encode_iceberg_snapshot_update_kind(kind: SnapshotUpdateKind) -> i32 { + (match kind { + SnapshotUpdateKind::FastAppend => r#gen::IcebergSnapshotUpdateKind::FastAppend, + SnapshotUpdateKind::FullOverwrite => r#gen::IcebergSnapshotUpdateKind::FullOverwrite, + SnapshotUpdateKind::RowDelta => r#gen::IcebergSnapshotUpdateKind::RowDelta, + }) as i32 + } + fn try_decode_json(&self, value: &str, description: &str) -> Result { serde_json::from_str(value) .map_err(|e| plan_datafusion_err!("failed to decode {description}: {e}")) } - fn try_encode_json(&self, value: &T, description: &str) -> Result { + fn try_encode_json( + &self, + value: &T, + description: &str, + ) -> Result { serde_json::to_string(value) .map_err(|e| plan_datafusion_err!("failed to encode {description}: {e}")) } @@ -3389,22 +4067,21 @@ impl RemoteExecutionCodec { fn try_decode_catalog_partition_field( &self, - field: &gen::CatalogPartitionFieldNode, + field: &r#gen::CatalogPartitionFieldNode, ) -> Result { - let transform_kind = gen::PartitionTransformKind::try_from(field.transform_kind) + let transform_kind = r#gen::PartitionTransformKind::try_from(field.transform_kind) .map_err(|_| plan_datafusion_err!("invalid partition transform kind"))?; let transform = match transform_kind { - gen::PartitionTransformKind::Unspecified | gen::PartitionTransformKind::Identity => { - None - } - gen::PartitionTransformKind::Year => Some(PartitionTransform::Year), - gen::PartitionTransformKind::Month => Some(PartitionTransform::Month), - gen::PartitionTransformKind::Day => Some(PartitionTransform::Day), - gen::PartitionTransformKind::Hour => Some(PartitionTransform::Hour), - gen::PartitionTransformKind::Bucket => { + r#gen::PartitionTransformKind::Unspecified + | r#gen::PartitionTransformKind::Identity => None, + r#gen::PartitionTransformKind::Year => Some(PartitionTransform::Year), + r#gen::PartitionTransformKind::Month => Some(PartitionTransform::Month), + r#gen::PartitionTransformKind::Day => Some(PartitionTransform::Day), + r#gen::PartitionTransformKind::Hour => Some(PartitionTransform::Hour), + r#gen::PartitionTransformKind::Bucket => { Some(PartitionTransform::Bucket(field.transform_value)) } - gen::PartitionTransformKind::Truncate => { + r#gen::PartitionTransformKind::Truncate => { Some(PartitionTransform::Truncate(field.transform_value)) } }; @@ -3417,20 +4094,24 @@ impl RemoteExecutionCodec { fn try_encode_catalog_partition_field( &self, field: &CatalogPartitionField, - ) -> Result { + ) -> Result { let (transform_kind, transform_value) = match field.transform { - None => (gen::PartitionTransformKind::Unspecified as i32, 0), - Some(PartitionTransform::Identity) => (gen::PartitionTransformKind::Identity as i32, 0), - Some(PartitionTransform::Year) => (gen::PartitionTransformKind::Year as i32, 0), - Some(PartitionTransform::Month) => (gen::PartitionTransformKind::Month as i32, 0), - Some(PartitionTransform::Day) => (gen::PartitionTransformKind::Day as i32, 0), - Some(PartitionTransform::Hour) => (gen::PartitionTransformKind::Hour as i32, 0), - Some(PartitionTransform::Bucket(n)) => (gen::PartitionTransformKind::Bucket as i32, n), + None => (r#gen::PartitionTransformKind::Unspecified as i32, 0), + Some(PartitionTransform::Identity) => { + (r#gen::PartitionTransformKind::Identity as i32, 0) + } + Some(PartitionTransform::Year) => (r#gen::PartitionTransformKind::Year as i32, 0), + Some(PartitionTransform::Month) => (r#gen::PartitionTransformKind::Month as i32, 0), + Some(PartitionTransform::Day) => (r#gen::PartitionTransformKind::Day as i32, 0), + Some(PartitionTransform::Hour) => (r#gen::PartitionTransformKind::Hour as i32, 0), + Some(PartitionTransform::Bucket(n)) => { + (r#gen::PartitionTransformKind::Bucket as i32, n) + } Some(PartitionTransform::Truncate(w)) => { - (gen::PartitionTransformKind::Truncate as i32, w) + (r#gen::PartitionTransformKind::Truncate as i32, w) } }; - Ok(gen::CatalogPartitionFieldNode { + Ok(r#gen::CatalogPartitionFieldNode { column: field.column.clone(), transform_kind, transform_value, @@ -3444,7 +4125,7 @@ impl RemoteExecutionCodec { None => return plan_err!("ExtendedStreamUdf: no UDF found"), }; let udf: Arc = match stream_udf_kind { - StreamUdfKind::PySparkMapIter(gen::PySparkMapIterUdf { + StreamUdfKind::PySparkMapIter(r#gen::PySparkMapIterUdf { kind, name, payload, @@ -3467,7 +4148,7 @@ impl RemoteExecutionCodec { Arc::new(config), )) } - StreamUdfKind::PySparkUdtf(gen::PySparkUdtf { + StreamUdfKind::PySparkUdtf(r#gen::PySparkUdtf { kind, name, payload, @@ -3517,7 +4198,7 @@ impl RemoteExecutionCodec { let kind = self.try_encode_pyspark_map_iter_kind(func.kind())?; let output_schema = try_encode_schema(func.output_schema().as_ref())?; let config = self.try_encode_pyspark_udf_config(func.config())?; - StreamUdfKind::PySparkMapIter(gen::PySparkMapIterUdf { + StreamUdfKind::PySparkMapIter(r#gen::PySparkMapIterUdf { kind, name: func.name().to_string(), payload: func.payload().to_vec(), @@ -3539,7 +4220,7 @@ impl RemoteExecutionCodec { .map(|x| x.to_vec()) .unwrap_or_default(); let config = self.try_encode_pyspark_udf_config(func.config())?; - StreamUdfKind::PySparkUdtf(gen::PySparkUdtf { + StreamUdfKind::PySparkUdtf(r#gen::PySparkUdtf { kind, name: func.name().to_string(), payload: func.payload().to_vec(), @@ -3583,7 +4264,7 @@ impl RemoteExecutionCodec { fn try_decode_lex_ordering( &self, - lex_ordering: &gen::LexOrdering, + lex_ordering: &r#gen::LexOrdering, schema: &Schema, ctx: &TaskContext, ) -> Result { @@ -3607,7 +4288,7 @@ impl RemoteExecutionCodec { } } - fn try_encode_lex_ordering(&self, lex_ordering: &LexOrdering) -> Result { + fn try_encode_lex_ordering(&self, lex_ordering: &LexOrdering) -> Result { let lex_ordering = serialize_physical_sort_exprs( lex_ordering.to_vec(), self, @@ -3617,14 +4298,14 @@ impl RemoteExecutionCodec { .into_iter() .map(try_encode_message) .collect::>()?; - Ok(gen::LexOrdering { + Ok(r#gen::LexOrdering { values: lex_ordering, }) } fn try_decode_lex_orderings( &self, - lex_orderings: &[gen::LexOrdering], + lex_orderings: &[r#gen::LexOrdering], schema: &Schema, ctx: &TaskContext, ) -> Result> { @@ -3639,7 +4320,7 @@ impl RemoteExecutionCodec { fn try_encode_lex_orderings( &self, lex_orderings: &[LexOrdering], - ) -> Result> { + ) -> Result> { let mut result = vec![]; for lex_ordering in lex_orderings { let lex_ordering = self.try_encode_lex_ordering(lex_ordering)?; @@ -3648,43 +4329,43 @@ impl RemoteExecutionCodec { Ok(result) } - fn try_decode_constraint(&self, constraint: &gen::Constraint) -> Result { - let gen::Constraint { kind: Some(kind) } = constraint else { + fn try_decode_constraint(&self, constraint: &r#gen::Constraint) -> Result { + let r#gen::Constraint { kind: Some(kind) } = constraint else { return plan_err!("missing constraint kind"); }; match kind { - gen::constraint::Kind::PrimaryKey(gen::PrimaryKeyConstraint { indices }) => { + r#gen::constraint::Kind::PrimaryKey(r#gen::PrimaryKeyConstraint { indices }) => { let indices = indices.iter().map(|x| *x as usize).collect(); Ok(Constraint::PrimaryKey(indices)) } - gen::constraint::Kind::Unique(gen::UniqueConstraint { indices }) => { + r#gen::constraint::Kind::Unique(r#gen::UniqueConstraint { indices }) => { let indices = indices.iter().map(|x| *x as usize).collect(); Ok(Constraint::Unique(indices)) } } } - fn try_encode_constraint(&self, constraint: &Constraint) -> Result { + fn try_encode_constraint(&self, constraint: &Constraint) -> Result { let kind = match constraint { Constraint::PrimaryKey(indices) => { let indices = indices.iter().map(|x| *x as u64).collect(); - gen::constraint::Kind::PrimaryKey(gen::PrimaryKeyConstraint { indices }) + r#gen::constraint::Kind::PrimaryKey(r#gen::PrimaryKeyConstraint { indices }) } Constraint::Unique(indices) => { let indices = indices.iter().map(|x| *x as u64).collect(); - gen::constraint::Kind::Unique(gen::UniqueConstraint { indices }) + r#gen::constraint::Kind::Unique(r#gen::UniqueConstraint { indices }) } }; - Ok(gen::Constraint { kind: Some(kind) }) + Ok(r#gen::Constraint { kind: Some(kind) }) } fn try_decode_equivalence_class( &self, - class: &gen::EquivalenceClass, + class: &r#gen::EquivalenceClass, schema: &Schema, ctx: &TaskContext, ) -> Result { - let gen::EquivalenceClass { exprs } = class; + let r#gen::EquivalenceClass { exprs } = class; let exprs = exprs .iter() .map(|expr| try_decode_physical_expr(ctx, self, expr, schema)) @@ -3696,21 +4377,21 @@ impl RemoteExecutionCodec { fn try_encode_equivalence_class( &self, class: &EquivalenceClass, - ) -> Result { + ) -> Result { let exprs = class .iter() .map(|expr| try_encode_physical_expr(self, expr)) .collect::>>()?; - Ok(gen::EquivalenceClass { exprs }) + Ok(r#gen::EquivalenceClass { exprs }) } fn try_decode_constant_expression( &self, - const_expr: &gen::ConstantExpr, + const_expr: &r#gen::ConstantExpr, schema: &Schema, ctx: &TaskContext, ) -> Result { - let gen::ConstantExpr { + let r#gen::ConstantExpr { expr, across_partitions, } = const_expr; @@ -3722,11 +4403,14 @@ impl RemoteExecutionCodec { Ok(ConstExpr::new(expr, across_partitions)) } - fn try_encode_constant_expression(&self, const_expr: &ConstExpr) -> Result { + fn try_encode_constant_expression( + &self, + const_expr: &ConstExpr, + ) -> Result { let expr = try_encode_physical_expr(self, &const_expr.expr)?; let across_partitions = self.try_encode_constant_across_partitions(&const_expr.across_partitions)?; - Ok(gen::ConstantExpr { + Ok(r#gen::ConstantExpr { expr, across_partitions: Some(across_partitions), }) @@ -3734,51 +4418,53 @@ impl RemoteExecutionCodec { fn try_decode_constant_across_partitions( &self, - constant: &gen::ConstantAcrossPartitions, + constant: &r#gen::ConstantAcrossPartitions, ) -> Result { - let gen::ConstantAcrossPartitions { kind: Some(kind) } = constant else { + let r#gen::ConstantAcrossPartitions { kind: Some(kind) } = constant else { return plan_err!("missing constant across partitions kind"); }; match kind { - gen::constant_across_partitions::Kind::Uniform(gen::UniformConstant { value }) => { + r#gen::constant_across_partitions::Kind::Uniform(r#gen::UniformConstant { value }) => { let value = value .as_ref() .map(|x| self.try_decode_scalar_value(x)) .transpose()?; Ok(AcrossPartitions::Uniform(value)) } - gen::constant_across_partitions::Kind::Heterogeneous(gen::HeterogeneousConstant {}) => { - Ok(AcrossPartitions::Heterogeneous) - } + r#gen::constant_across_partitions::Kind::Heterogeneous( + r#gen::HeterogeneousConstant {}, + ) => Ok(AcrossPartitions::Heterogeneous), } } fn try_encode_constant_across_partitions( &self, constant: &AcrossPartitions, - ) -> Result { + ) -> Result { let kind = match constant { AcrossPartitions::Uniform(value) => { let value = value .as_ref() .map(|x| self.try_encode_scalar_value(x)) .transpose()?; - gen::constant_across_partitions::Kind::Uniform(gen::UniformConstant { value }) + r#gen::constant_across_partitions::Kind::Uniform(r#gen::UniformConstant { value }) } AcrossPartitions::Heterogeneous => { - gen::constant_across_partitions::Kind::Heterogeneous(gen::HeterogeneousConstant {}) + r#gen::constant_across_partitions::Kind::Heterogeneous( + r#gen::HeterogeneousConstant {}, + ) } }; - Ok(gen::ConstantAcrossPartitions { kind: Some(kind) }) + Ok(r#gen::ConstantAcrossPartitions { kind: Some(kind) }) } fn try_decode_equivalence_group( &self, - eq_group: &gen::EquivalenceGroup, + eq_group: &r#gen::EquivalenceGroup, schema: &Schema, ctx: &TaskContext, ) -> Result { - let gen::EquivalenceGroup { classes } = eq_group; + let r#gen::EquivalenceGroup { classes } = eq_group; let classes = classes .iter() .map(|class| self.try_decode_equivalence_class(class, schema, ctx)) @@ -3789,20 +4475,20 @@ impl RemoteExecutionCodec { fn try_encode_equivalence_group( &self, eq_group: &EquivalenceGroup, - ) -> Result { + ) -> Result { let classes = eq_group .iter() .map(|class| self.try_encode_equivalence_class(class)) .collect::>>()?; - Ok(gen::EquivalenceGroup { classes }) + Ok(r#gen::EquivalenceGroup { classes }) } fn try_decode_equivalence_properties( &self, - eq_properties: &gen::EquivalenceProperties, + eq_properties: &r#gen::EquivalenceProperties, ctx: &TaskContext, ) -> Result { - let gen::EquivalenceProperties { + let r#gen::EquivalenceProperties { eq_group, constants, orderings, @@ -3834,7 +4520,7 @@ impl RemoteExecutionCodec { fn try_encode_equivalence_properties( &self, eq_properties: &EquivalenceProperties, - ) -> Result { + ) -> Result { let schema = try_encode_schema(eq_properties.schema().as_ref())?; let eq_group = self.try_encode_equivalence_group(eq_properties.eq_group())?; let constants = eq_properties @@ -3848,7 +4534,7 @@ impl RemoteExecutionCodec { .iter() .map(|x| self.try_encode_constraint(x)) .collect::>>()?; - Ok(gen::EquivalenceProperties { + Ok(r#gen::EquivalenceProperties { eq_group: Some(eq_group), constants, orderings, @@ -3858,110 +4544,110 @@ impl RemoteExecutionCodec { } fn try_decode_show_string_style(&self, style: i32) -> Result { - let style = gen::ShowStringStyle::try_from(style) + let style = r#gen::ShowStringStyle::try_from(style) .map_err(|e| plan_datafusion_err!("failed to decode style: {e}"))?; let style = match style { - gen::ShowStringStyle::Default => ShowStringStyle::Default, - gen::ShowStringStyle::Vertical => ShowStringStyle::Vertical, - gen::ShowStringStyle::Html => ShowStringStyle::Html, + r#gen::ShowStringStyle::Default => ShowStringStyle::Default, + r#gen::ShowStringStyle::Vertical => ShowStringStyle::Vertical, + r#gen::ShowStringStyle::Html => ShowStringStyle::Html, }; Ok(style) } fn try_encode_show_string_style(&self, style: ShowStringStyle) -> Result { let style = match style { - ShowStringStyle::Default => gen::ShowStringStyle::Default, - ShowStringStyle::Vertical => gen::ShowStringStyle::Vertical, - ShowStringStyle::Html => gen::ShowStringStyle::Html, + ShowStringStyle::Default => r#gen::ShowStringStyle::Default, + ShowStringStyle::Vertical => r#gen::ShowStringStyle::Vertical, + ShowStringStyle::Html => r#gen::ShowStringStyle::Html, }; Ok(style as i32) } fn try_decode_pyspark_udf_kind(&self, kind: i32) -> Result { - let kind = gen::PySparkUdfKind::try_from(kind) + let kind = r#gen::PySparkUdfKind::try_from(kind) .map_err(|e| plan_datafusion_err!("failed to decode pyspark UDF kind: {e}"))?; let kind = match kind { - gen::PySparkUdfKind::Batch => PySparkUdfKind::Batch, - gen::PySparkUdfKind::ArrowBatch => PySparkUdfKind::ArrowBatch, - gen::PySparkUdfKind::ScalarPandas => PySparkUdfKind::ScalarPandas, - gen::PySparkUdfKind::ScalarPandasIter => PySparkUdfKind::ScalarPandasIter, + r#gen::PySparkUdfKind::Batch => PySparkUdfKind::Batch, + r#gen::PySparkUdfKind::ArrowBatch => PySparkUdfKind::ArrowBatch, + r#gen::PySparkUdfKind::ScalarPandas => PySparkUdfKind::ScalarPandas, + r#gen::PySparkUdfKind::ScalarPandasIter => PySparkUdfKind::ScalarPandasIter, // Spark 4.0 Arrow-native scalar UDF kinds - gen::PySparkUdfKind::ScalarArrow => PySparkUdfKind::ScalarArrow, - gen::PySparkUdfKind::ScalarArrowIter => PySparkUdfKind::ScalarArrowIter, + r#gen::PySparkUdfKind::ScalarArrow => PySparkUdfKind::ScalarArrow, + r#gen::PySparkUdfKind::ScalarArrowIter => PySparkUdfKind::ScalarArrowIter, }; Ok(kind) } fn try_encode_pyspark_udf_kind(&self, kind: PySparkUdfKind) -> Result { let kind = match kind { - PySparkUdfKind::Batch => gen::PySparkUdfKind::Batch, - PySparkUdfKind::ArrowBatch => gen::PySparkUdfKind::ArrowBatch, - PySparkUdfKind::ScalarPandas => gen::PySparkUdfKind::ScalarPandas, - PySparkUdfKind::ScalarPandasIter => gen::PySparkUdfKind::ScalarPandasIter, - PySparkUdfKind::ScalarArrow => gen::PySparkUdfKind::ScalarArrow, - PySparkUdfKind::ScalarArrowIter => gen::PySparkUdfKind::ScalarArrowIter, + PySparkUdfKind::Batch => r#gen::PySparkUdfKind::Batch, + PySparkUdfKind::ArrowBatch => r#gen::PySparkUdfKind::ArrowBatch, + PySparkUdfKind::ScalarPandas => r#gen::PySparkUdfKind::ScalarPandas, + PySparkUdfKind::ScalarPandasIter => r#gen::PySparkUdfKind::ScalarPandasIter, + PySparkUdfKind::ScalarArrow => r#gen::PySparkUdfKind::ScalarArrow, + PySparkUdfKind::ScalarArrowIter => r#gen::PySparkUdfKind::ScalarArrowIter, }; Ok(kind as i32) } // Decode/encode grouped aggregate UDF kind (Pandas vs Arrow) fn try_decode_pyspark_group_agg_kind(&self, kind: i32) -> Result { - let kind = gen::PySparkGroupAggKind::try_from(kind) + let kind = r#gen::PySparkGroupAggKind::try_from(kind) .map_err(|e| plan_datafusion_err!("failed to decode pyspark group agg kind: {e}"))?; let kind = match kind { - gen::PySparkGroupAggKind::Pandas => PySparkGroupAggKind::Pandas, - gen::PySparkGroupAggKind::Arrow => PySparkGroupAggKind::Arrow, + r#gen::PySparkGroupAggKind::Pandas => PySparkGroupAggKind::Pandas, + r#gen::PySparkGroupAggKind::Arrow => PySparkGroupAggKind::Arrow, }; Ok(kind) } fn try_encode_pyspark_group_agg_kind(&self, kind: PySparkGroupAggKind) -> Result { let kind = match kind { - PySparkGroupAggKind::Pandas => gen::PySparkGroupAggKind::Pandas, - PySparkGroupAggKind::Arrow => gen::PySparkGroupAggKind::Arrow, + PySparkGroupAggKind::Pandas => r#gen::PySparkGroupAggKind::Pandas, + PySparkGroupAggKind::Arrow => r#gen::PySparkGroupAggKind::Arrow, }; Ok(kind as i32) } fn try_decode_pyspark_map_iter_kind(&self, kind: i32) -> Result { - let kind = gen::PySparkMapIterKind::try_from(kind) + let kind = r#gen::PySparkMapIterKind::try_from(kind) .map_err(|e| plan_datafusion_err!("failed to decode pyspark map iter kind: {e}"))?; let kind = match kind { - gen::PySparkMapIterKind::Arrow => PySparkMapIterKind::Arrow, - gen::PySparkMapIterKind::Pandas => PySparkMapIterKind::Pandas, + r#gen::PySparkMapIterKind::Arrow => PySparkMapIterKind::Arrow, + r#gen::PySparkMapIterKind::Pandas => PySparkMapIterKind::Pandas, }; Ok(kind) } fn try_encode_pyspark_map_iter_kind(&self, kind: PySparkMapIterKind) -> Result { let kind = match kind { - PySparkMapIterKind::Arrow => gen::PySparkMapIterKind::Arrow, - PySparkMapIterKind::Pandas => gen::PySparkMapIterKind::Pandas, + PySparkMapIterKind::Arrow => r#gen::PySparkMapIterKind::Arrow, + PySparkMapIterKind::Pandas => r#gen::PySparkMapIterKind::Pandas, }; Ok(kind as i32) } fn try_decode_pyspark_udtf_kind(&self, kind: i32) -> Result { - let kind = gen::PySparkUdtfKind::try_from(kind) + let kind = r#gen::PySparkUdtfKind::try_from(kind) .map_err(|e| plan_datafusion_err!("failed to decode pyspark UDTF kind: {e}"))?; let kind = match kind { - gen::PySparkUdtfKind::Table => PySparkUdtfKind::Table, - gen::PySparkUdtfKind::ArrowTable => PySparkUdtfKind::ArrowTable, + r#gen::PySparkUdtfKind::Table => PySparkUdtfKind::Table, + r#gen::PySparkUdtfKind::ArrowTable => PySparkUdtfKind::ArrowTable, }; Ok(kind) } fn try_encode_pyspark_udtf_kind(&self, kind: PySparkUdtfKind) -> Result { let kind = match kind { - PySparkUdtfKind::Table => gen::PySparkUdtfKind::Table, - PySparkUdtfKind::ArrowTable => gen::PySparkUdtfKind::ArrowTable, + PySparkUdtfKind::Table => r#gen::PySparkUdtfKind::Table, + PySparkUdtfKind::ArrowTable => r#gen::PySparkUdtfKind::ArrowTable, }; Ok(kind as i32) } fn try_decode_pyspark_udf_config( &self, - config: &gen::PySparkUdfConfig, + config: &r#gen::PySparkUdfConfig, ) -> Result { let config = PySparkUdfConfig { session_timezone: config.session_timezone.clone(), @@ -3975,6 +4661,9 @@ impl RemoteExecutionCodec { python_udf_pandas_int_to_decimal_coercion_enabled: config .python_udf_pandas_int_to_decimal_coercion_enabled, binary_as_bytes: config.binary_as_bytes, + python_udf_pandas_prefer_int_extension_dtype: config + .python_udf_pandas_prefer_int_extension_dtype, + arrow_use_large_var_types: config.arrow_use_large_var_types, }; Ok(config) } @@ -3982,8 +4671,8 @@ impl RemoteExecutionCodec { fn try_encode_pyspark_udf_config( &self, config: &PySparkUdfConfig, - ) -> Result { - let config = gen::PySparkUdfConfig { + ) -> Result { + let config = r#gen::PySparkUdfConfig { session_timezone: config.session_timezone.clone(), pandas_window_bound_types: config.pandas_window_bound_types.clone(), pandas_grouped_map_assign_columns_by_name: config @@ -3995,6 +4684,9 @@ impl RemoteExecutionCodec { python_udf_pandas_int_to_decimal_coercion_enabled: config .python_udf_pandas_int_to_decimal_coercion_enabled, binary_as_bytes: config.binary_as_bytes, + python_udf_pandas_prefer_int_extension_dtype: config + .python_udf_pandas_prefer_int_extension_dtype, + arrow_use_large_var_types: config.arrow_use_large_var_types, }; Ok(config) } @@ -4004,14 +4696,14 @@ impl RemoteExecutionCodec { } fn try_decode_compression_type_variant(&self, variant: i32) -> Result { - let variant = gen::CompressionTypeVariant::try_from(variant) + let variant = r#gen::CompressionTypeVariant::try_from(variant) .map_err(|e| plan_datafusion_err!("failed to decode compression type variant: {e}"))?; let variant = match variant { - gen::CompressionTypeVariant::Gzip => CompressionTypeVariant::GZIP, - gen::CompressionTypeVariant::Bzip2 => CompressionTypeVariant::BZIP2, - gen::CompressionTypeVariant::Xz => CompressionTypeVariant::XZ, - gen::CompressionTypeVariant::Zstd => CompressionTypeVariant::ZSTD, - gen::CompressionTypeVariant::Uncompressed => CompressionTypeVariant::UNCOMPRESSED, + r#gen::CompressionTypeVariant::Gzip => CompressionTypeVariant::GZIP, + r#gen::CompressionTypeVariant::Bzip2 => CompressionTypeVariant::BZIP2, + r#gen::CompressionTypeVariant::Xz => CompressionTypeVariant::XZ, + r#gen::CompressionTypeVariant::Zstd => CompressionTypeVariant::ZSTD, + r#gen::CompressionTypeVariant::Uncompressed => CompressionTypeVariant::UNCOMPRESSED, }; Ok(variant) } @@ -4026,11 +4718,11 @@ impl RemoteExecutionCodec { fn try_encode_compression_type_variant(&self, variant: CompressionTypeVariant) -> Result { let variant = match variant { - CompressionTypeVariant::GZIP => gen::CompressionTypeVariant::Gzip, - CompressionTypeVariant::BZIP2 => gen::CompressionTypeVariant::Bzip2, - CompressionTypeVariant::XZ => gen::CompressionTypeVariant::Xz, - CompressionTypeVariant::ZSTD => gen::CompressionTypeVariant::Zstd, - CompressionTypeVariant::UNCOMPRESSED => gen::CompressionTypeVariant::Uncompressed, + CompressionTypeVariant::GZIP => r#gen::CompressionTypeVariant::Gzip, + CompressionTypeVariant::BZIP2 => r#gen::CompressionTypeVariant::Bzip2, + CompressionTypeVariant::XZ => r#gen::CompressionTypeVariant::Xz, + CompressionTypeVariant::ZSTD => r#gen::CompressionTypeVariant::Zstd, + CompressionTypeVariant::UNCOMPRESSED => r#gen::CompressionTypeVariant::Uncompressed, }; Ok(variant as i32) } @@ -4102,19 +4794,212 @@ mod tests { codec.try_decode_udf(&name, &buf) } + fn downcast_udf<'a, T: ScalarUDFImpl>(udf: &'a ScalarUDF, name: &str) -> Result<&'a T> { + udf.inner() + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded UDF should be {name}")) + } + + fn round_trip_udwf(udwf: WindowUDF) -> Result> { + round_trip_udwf_arc(Arc::new(udwf)) + } + + fn round_trip_udwf_arc(udwf: Arc) -> Result> { + let codec = RemoteExecutionCodec; + let name = udwf.name().to_string(); + let mut buf = vec![]; + codec.try_encode_udwf(udwf.as_ref(), &mut buf)?; + codec.try_decode_udwf(&name, &buf) + } + + #[test] + fn test_round_trip_recursive_query_preserves_output_schema() -> Result<()> { + use datafusion::arrow::datatypes::{DataType, Field}; + use datafusion::physical_plan::empty::EmptyExec; + + let static_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let output_schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + true, + )])); + let static_term: Arc = + Arc::new(EmptyExec::new(Arc::clone(&static_schema))); + let recursive_term: Arc = + Arc::new(EmptyExec::new(Arc::clone(&output_schema))); + let plan = Arc::new(RecursiveQueryExec::try_new( + "numbers".to_string(), + Arc::clone(&output_schema), + static_term, + recursive_term, + false, + )?); + + let codec = RemoteExecutionCodec; + let bytes = try_encode_physical_plan(&codec, plan)?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let recursive_query = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a RecursiveQueryExec"))?; + + assert_eq!(recursive_query.schema(), output_schema); + assert_eq!(recursive_query.static_term().schema(), output_schema); + assert_eq!(recursive_query.recursive_term().schema(), output_schema); + Ok(()) + } + + #[test] + fn test_struct_field_matching_proto_round_trip_and_default() -> Result<()> { + for mode in [ + StructFieldMatching::Name, + StructFieldMatching::PhysicalName, + StructFieldMatching::FieldId, + ] { + let encoded = RemoteExecutionCodec::try_encode_struct_field_matching(mode); + assert_eq!( + RemoteExecutionCodec::try_decode_struct_field_matching(encoded)?, + mode + ); + } + + assert_eq!( + RemoteExecutionCodec::try_decode_struct_field_matching( + r#gen::CastColumnExprNode::default().struct_field_matching, + )?, + StructFieldMatching::Name + ); + assert!(RemoteExecutionCodec::try_decode_struct_field_matching(i32::MAX).is_err()); + + for mode in [ + SchemaEvolutionTimezoneMode::Strict, + SchemaEvolutionTimezoneMode::Relaxed, + ] { + let encoded = RemoteExecutionCodec::try_encode_schema_evolution_timezone_mode(mode); + assert_eq!( + RemoteExecutionCodec::try_decode_schema_evolution_timezone_mode(encoded)?, + mode + ); + } + assert!(RemoteExecutionCodec::try_decode_schema_evolution_timezone_mode(i32::MAX).is_err()); + Ok(()) + } + + #[test] + fn test_round_trip_schema_evolution_cast_preserves_timezone_mode() -> Result<()> { + let input_field = Arc::new(Field::new( + "event_time", + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))), + true, + )); + let target_field = Arc::new(Field::new( + "event_time", + DataType::Timestamp( + TimeUnit::Microsecond, + Some(Arc::from("America/Los_Angeles")), + ), + true, + )); + let input_schema = Arc::new(Schema::new(vec![Arc::clone(&input_field)])); + let expression = Arc::new(SchemaEvolutionCastColumnExpr::new_relaxed_timezone( + Arc::new(Column::new("event_time", 0)), + input_field, + target_field, + None, + StructFieldMatching::PhysicalName, + )) as Arc; + + let decoded = round_trip_expr(&expression, input_schema.as_ref())?; + let decoded = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded expression is not a schema cast"))?; + + assert_eq!(decoded.matching(), StructFieldMatching::PhysicalName); + assert_eq!( + decoded.timezone_mode(), + SchemaEvolutionTimezoneMode::Relaxed + ); + Ok(()) + } + #[test] fn test_round_trip_spark_variant_explode_helper_udf() -> Result<()> { let decoded = round_trip_udf(ScalarUDF::from(SparkVariantExplodeUdf::new()))?; - assert!(decoded - .inner() - .downcast_ref::() - .is_some()); + assert!( + decoded + .inner() + .downcast_ref::() + .is_some() + ); assert_eq!(decoded.name(), "spark_variant_explode"); Ok(()) } + #[test] + fn test_round_trip_spark_date_part_udf() -> Result<()> { + let decoded = round_trip_udf(ScalarUDF::from(SparkDatePart::new()))?; + + assert!(decoded.inner().downcast_ref::().is_some()); + assert_eq!(decoded.name(), "date_part"); + + Ok(()) + } + + #[test] + fn test_round_trip_spark_first_last_value_udwf() -> Result<()> { + let decoded = round_trip_udwf(WindowUDF::from(SparkFirstLastValue::first(true)))?; + let func = decoded + .inner() + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded UDWF is not SparkFirstLastValue"))?; + assert_eq!(decoded.name(), "first_value"); + assert_eq!(func.kind(), SparkFirstLastValueKind::First); + assert!(func.ignore_nulls()); + + let decoded = round_trip_udwf(WindowUDF::from(SparkFirstLastValue::last(true)))?; + let func = decoded + .inner() + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded UDWF is not SparkFirstLastValue"))?; + assert_eq!(decoded.name(), "last_value"); + assert_eq!(func.kind(), SparkFirstLastValueKind::Last); + assert!(func.ignore_nulls()); + + Ok(()) + } + + #[test] + fn test_round_trip_standard_window_udwfs() -> Result<()> { + let udwfs = [ + cume_dist_udwf(), + dense_rank_udwf(), + first_value_udwf(), + lag_udwf(), + last_value_udwf(), + lead_udwf(), + nth_value_udwf(), + rank_udwf(), + row_number_udwf(), + percent_rank_udwf(), + ]; + + for udwf in udwfs { + let name = udwf.name().to_string(); + let decoded = round_trip_udwf_arc(udwf)?; + assert_eq!(decoded.name(), name); + } + + let decoded = round_trip_udwf(WindowUDF::from(SparkNtile::new()))?; + assert!(decoded.inner().downcast_ref::().is_some()); + + Ok(()) + } + fn round_trip_expr( expr: &Arc, schema: &Schema, @@ -4172,7 +5057,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; @@ -4229,7 +5114,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_exists::SparkArrayExists; @@ -4276,7 +5161,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_forall::SparkArrayForall; @@ -4321,7 +5206,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_aggregate::SparkArrayAggregate; @@ -4379,7 +5264,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_aggregate::SparkArrayAggregate; @@ -4467,6 +5352,124 @@ mod tests { assert_same_result(&physical, decoded_expr, schema_ref, vec![Arc::new(list)]) } + #[test] + fn test_round_trip_remote_checkpoint_write_plan() -> Result<()> { + use datafusion::physical_plan::empty::EmptyExec; + + let input_schema = Arc::new(Schema::new(vec![Field::new( + "logical_name", + DataType::Int64, + true, + )])); + let storage_schema = Arc::new(Schema::new(vec![Field::new("_c0", DataType::Int64, true)])); + let checkpoint = RemoteCheckpointWriteExec::try_new( + Arc::new(EmptyExec::new(input_schema)), + datafusion::execution::object_store::ObjectStoreUrl::parse("s3://checkpoint-bucket")?, + object_store::path::Path::from("checkpoints/session/relation"), + Arc::clone(&storage_schema), + )?; + let codec = RemoteExecutionCodec; + + let bytes = try_encode_physical_plan(&codec, Arc::new(checkpoint))?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let decoded = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a checkpoint writer"))?; + + assert_eq!( + decoded.object_store_url().as_str(), + "s3://checkpoint-bucket/" + ); + assert_eq!( + decoded.prefix(), + &object_store::path::Path::from("checkpoints/session/relation") + ); + assert_eq!(decoded.storage_schema(), &storage_schema); + assert_eq!(decoded.input().schema().field(0).name(), "logical_name"); + Ok(()) + } + + #[test] + fn test_round_trip_remote_checkpoint_commit_plan() -> Result<()> { + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_plan::empty::EmptyExec; + + let metadata_schema = Arc::new(Schema::new(vec![Field::new( + "metadata", + DataType::Utf8, + false, + )])); + let logical_schema = Arc::new(Schema::new(vec![Field::new( + "logical_name", + DataType::Int64, + true, + )])); + let storage_schema = Arc::new(Schema::new(vec![Field::new("_c0", DataType::Int64, true)])); + let key = Arc::new(Column::new("_c0", 0)) as Arc; + let ordering = LexOrdering::new([PhysicalSortExpr::new_default(Arc::clone(&key))]) + .ok_or_else(|| plan_datafusion_err!("expected non-empty checkpoint ordering"))?; + let checkpoint = RemoteCheckpointCommitExec::new( + Arc::new(EmptyExec::new(metadata_schema)), + "relation".to_string(), + datafusion::execution::object_store::ObjectStoreUrl::parse("s3://checkpoint-bucket")?, + object_store::path::Path::from("checkpoints/session/relation"), + Arc::clone(&logical_schema), + Arc::clone(&storage_schema), + Partitioning::Hash(vec![key], 4), + Some(ordering), + ); + let codec = RemoteExecutionCodec; + + let bytes = try_encode_physical_plan(&codec, Arc::new(checkpoint))?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + let decoded = decoded + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("decoded plan is not a checkpoint commit"))?; + + assert_eq!(decoded.relation_id(), "relation"); + assert_eq!(decoded.logical_schema(), &logical_schema); + assert_eq!(decoded.storage_schema(), &storage_schema); + let Partitioning::Hash(expressions, 4) = decoded.checkpoint_partitioning() else { + return plan_err!("expected checkpoint hash partitioning"); + }; + let partition_column = expressions[0] + .downcast_ref::() + .ok_or_else(|| plan_datafusion_err!("checkpoint hash key is not a column"))?; + assert_eq!(partition_column.name(), "_c0"); + assert_eq!(partition_column.index(), 0); + assert!(decoded.checkpoint_ordering().is_some()); + Ok(()) + } + + #[test] + fn test_round_trip_checkpoint_data_source_partitioning() -> Result<()> { + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_plan::ExecutionPlanProperties; + + let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Int64, false)])); + let source = MemorySourceConfig::try_new(&[vec![], vec![]], Arc::clone(&schema), None)?; + let key = Arc::new(Column::new("key", 0)) as Arc; + let source = Arc::new(CheckpointDataSource::new( + Arc::new(source), + Partitioning::Hash(vec![key], 2), + )); + let scan = DataSourceExec::new(source); + let remote_scan = RemoteDataSourceExec::new(&scan); + let codec = RemoteExecutionCodec; + + let bytes = try_encode_physical_plan(&codec, Arc::new(remote_scan))?; + let decoded = try_decode_physical_plan(&TaskContext::default(), &codec, &bytes)?; + + assert!(decoded.is::()); + assert!(matches!( + decoded.output_partitioning(), + Partitioning::Hash(_, 2) + )); + Ok(()) + } + #[test] fn test_hash_output_partitioning_decodes_higher_order_key() -> Result<()> { use crate::task::definition::{TaskOutput, TaskOutputDistribution, TaskOutputLocator}; @@ -4511,7 +5514,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; @@ -4568,7 +5571,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; @@ -4625,7 +5628,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Case, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Case, Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_sort::SparkArraySort; @@ -4684,7 +5687,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_sort::SparkArraySort; @@ -4736,7 +5739,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; @@ -4799,7 +5802,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; @@ -4842,7 +5845,6 @@ mod tests { count_hofs(&decoded)? >= 2, "expected at least 2 HigherOrderFunctionExpr nodes" ); - assert_same_result(&physical, &decoded, schema_ref, vec![Arc::new(list)]) } @@ -4866,7 +5868,7 @@ mod tests { use datafusion::common::DFSchema; use datafusion::logical_expr::execution_props::ExecutionProps; use datafusion::logical_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion::logical_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion::logical_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion::physical_expr::create_physical_expr; use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; @@ -4921,7 +5923,72 @@ mod tests { count_hofs(&decoded)? >= 2, "expected at least 2 HigherOrderFunctionExpr nodes" ); - assert_same_result(&physical, &decoded, schema_ref, vec![Arc::new(arr2d)]) } + + #[test] + fn test_round_trip_spark_timestamp_preserves_options() -> Result<()> { + let udf = SparkTimestamp::try_new(Some(Arc::from("America/Los_Angeles")), true, true)?; + let decoded = round_trip_udf(ScalarUDF::from(udf))?; + + let decoded = downcast_udf::(&decoded, "SparkTimestamp")?; + assert_eq!(decoded.timezone(), Some("America/Los_Angeles")); + assert!(decoded.is_try()); + + Ok(()) + } + + #[test] + fn test_round_trip_spark_date_preserves_options() -> Result<()> { + let decoded = round_trip_udf(ScalarUDF::from(SparkDate::new(true)))?; + + let decoded = downcast_udf::(&decoded, "SparkDate")?; + assert!(decoded.is_try()); + + Ok(()) + } + + #[test] + fn test_round_trip_spark_try_to_timestamp_preserves_options() -> Result<()> { + let decoded = round_trip_udf(ScalarUDF::from(SparkTryToTimestamp::try_new(Some( + Arc::from("America/Los_Angeles"), + ))))?; + + let decoded = downcast_udf::(&decoded, "SparkTryToTimestamp")?; + assert_eq!(decoded.timezone(), Some("America/Los_Angeles")); + + Ok(()) + } + + #[test] + fn test_round_trip_spark_unix_timestamp_preserves_options() -> Result<()> { + let decoded = round_trip_udf(ScalarUDF::from(SparkUnixTimestamp::new( + Arc::from("America/Los_Angeles"), + true, + )))?; + + let decoded = downcast_udf::(&decoded, "SparkUnixTimestamp")?; + assert_eq!(decoded.session_timezone(), "America/Los_Angeles"); + assert!(decoded.ansi_mode()); + + Ok(()) + } + + #[test] + fn test_round_trip_spark_date_format_round_trip() -> Result<()> { + let decoded = round_trip_udf(ScalarUDF::from(SparkDateFormat::new(Arc::from("UTC"))))?; + + assert!(decoded.inner().downcast_ref::().is_some()); + Ok(()) + } + + #[test] + fn test_round_trip_spark_date_part_standard_udf() -> Result<()> { + let decoded = round_trip_udf(ScalarUDF::from(SparkDatePart::new()))?; + + assert!(decoded.inner().downcast_ref::().is_some()); + assert_eq!(decoded.name(), "date_part"); + + Ok(()) + } } diff --git a/crates/sail-execution/src/proto/converter.rs b/crates/sail-execution/src/proto/converter.rs index ff2e2f245..5d44b3f8b 100644 --- a/crates/sail-execution/src/proto/converter.rs +++ b/crates/sail-execution/src/proto/converter.rs @@ -2,7 +2,7 @@ use std::fmt::{Debug, Formatter}; use std::sync::Arc; use datafusion::arrow::datatypes::{FieldRef, Schema}; -use datafusion::common::{plan_datafusion_err, plan_err, Result}; +use datafusion::common::{Result, plan_datafusion_err, plan_err}; use datafusion::logical_expr::{LambdaParametersProgress, ValueOrLambda}; use datafusion::physical_expr::expressions::{LambdaExpr, LambdaVariable}; use datafusion::physical_expr::{HigherOrderFunctionExpr, PhysicalExpr}; @@ -12,18 +12,18 @@ use datafusion_proto::physical_plan::{ PhysicalExtensionCodec, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension, }; use datafusion_proto::protobuf::{ - physical_expr_node, PhysicalExprNode, PhysicalExtensionExprNode, PhysicalPlanNode, + PhysicalExprNode, PhysicalExtensionExprNode, PhysicalPlanNode, physical_expr_node, }; use prost::Message; -use crate::plan::gen::extended_physical_expr_node::ExprKind; -use crate::plan::gen::{ +use crate::plan::r#gen::extended_physical_expr_node::ExprKind; +use crate::plan::r#gen::{ ExtendedPhysicalExprNode, HigherOrderUdfExprNode, LambdaExprNode, LambdaVariableExprNode, }; -use crate::proto::decode::try_decode_higher_order_udf; +use crate::proto::decode::{try_decode_field_ref, try_decode_higher_order_udf}; use crate::proto::encode::{try_encode_field_ref, try_encode_higher_order_udf}; -pub struct RemotePhysicalProtoConverter; +pub(super) struct RemotePhysicalProtoConverter; impl Debug for RemotePhysicalProtoConverter { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { @@ -59,22 +59,13 @@ impl PhysicalProtoConverterExtension for RemotePhysicalProtoConverter { self.higher_order_proto_to_expr(node, inputs, input_schema, ctx) } Some((ExprKind::LambdaVariable(node), _)) => { + let field = try_decode_field_ref(&node.field)?; let index = usize::try_from(node.index).map_err(|_| { plan_datafusion_err!( "LambdaVariable index {} does not fit in usize", node.index ) })?; - let field = input_schema - .fields() - .get(index) - .ok_or_else(|| { - plan_datafusion_err!( - "LambdaVariable index {index} out of bounds for schema with {} fields", - input_schema.fields().len() - ) - })? - .clone(); Ok(Arc::new(LambdaVariable::new(index, field))) } Some((ExprKind::Lambda(node), inputs)) => { @@ -185,7 +176,7 @@ impl RemotePhysicalProtoConverter { let param_sets = match fun.lambda_parameters(0, &value_or_lambda)? { LambdaParametersProgress::Complete(params) => params, LambdaParametersProgress::Partial(_) => { - return plan_err!("`{}` returned partial lambda parameters", fun.name()) + return plan_err!("`{}` returned partial lambda parameters", fun.name()); } }; diff --git a/crates/sail-execution/src/proto/decode.rs b/crates/sail-execution/src/proto/decode.rs index a987ee4f5..bb9effea8 100644 --- a/crates/sail-execution/src/proto/decode.rs +++ b/crates/sail-execution/src/proto/decode.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{Field, FieldRef, Schema}; -use datafusion::common::{plan_datafusion_err, Result}; +use datafusion::common::{Result, plan_datafusion_err}; use datafusion::execution::TaskContext; use datafusion::logical_expr::HigherOrderUDF; use datafusion::physical_expr::PhysicalExpr; @@ -19,11 +19,31 @@ use sail_function::scalar::array::spark_array_forall::SparkArrayForall; use sail_function::scalar::array::spark_array_sort::SparkArraySort; use sail_function::scalar::array::spark_array_transform::SparkArrayTransform; -use crate::plan::gen; -use crate::plan::gen::higher_order_udf::HigherOrderUdfKind; +use crate::plan::r#gen; +use crate::plan::r#gen::higher_order_udf::HigherOrderUdfKind; use crate::proto::converter::RemotePhysicalProtoConverter; -pub fn try_decode_message(buf: &[u8]) -> Result +pub fn decode_remote_physical_plan( + ctx: &TaskContext, + codec: &dyn PhysicalExtensionCodec, + buf: &[u8], +) -> Result> { + // The remote codec decodes data sources directly into `DataSourceExec`. + // `RemoteDataSourceExec` is only used before encoding to bypass DataFusion's + // built-in `DataSourceExec` codec. + try_decode_physical_plan(ctx, codec, buf) +} + +pub fn decode_remote_physical_expr( + ctx: &TaskContext, + codec: &dyn PhysicalExtensionCodec, + buf: &[u8], + schema: &Schema, +) -> Result> { + try_decode_physical_expr(ctx, codec, buf, schema) +} + +pub(super) fn try_decode_message(buf: &[u8]) -> Result where M: Message + Default, { @@ -32,18 +52,18 @@ where Ok(message) } -pub fn try_decode_schema(buf: &[u8]) -> Result { +pub(super) fn try_decode_schema(buf: &[u8]) -> Result { let schema = try_decode_message::(buf)?; Ok((&schema).try_into()?) } -pub fn try_decode_field_ref(buf: &[u8]) -> Result { +pub(super) fn try_decode_field_ref(buf: &[u8]) -> Result { let field = try_decode_message::(buf)?; let field: Field = (&field).try_into()?; Ok(Arc::new(field)) } -pub fn try_decode_physical_plan( +pub(super) fn try_decode_physical_plan( ctx: &TaskContext, codec: &dyn PhysicalExtensionCodec, buf: &[u8], @@ -52,7 +72,7 @@ pub fn try_decode_physical_plan( proto_to_physical_plan(ctx, codec, &plan) } -pub fn proto_to_physical_plan( +pub(super) fn proto_to_physical_plan( ctx: &TaskContext, codec: &dyn PhysicalExtensionCodec, plan: &PhysicalPlanNode, @@ -60,7 +80,7 @@ pub fn proto_to_physical_plan( plan.try_into_physical_plan_with_converter(ctx, codec, &RemotePhysicalProtoConverter {}) } -pub fn try_decode_physical_expr( +pub(super) fn try_decode_physical_expr( ctx: &TaskContext, codec: &dyn PhysicalExtensionCodec, buf: &[u8], @@ -70,7 +90,7 @@ pub fn try_decode_physical_expr( proto_to_physical_expr(ctx, codec, &expr, schema) } -pub fn proto_to_physical_expr( +pub(super) fn proto_to_physical_expr( ctx: &TaskContext, codec: &dyn PhysicalExtensionCodec, expr: &PhysicalExprNode, @@ -80,14 +100,16 @@ pub fn proto_to_physical_expr( converter.proto_to_physical_expr(expr, schema, &PhysicalPlanDecodeContext::new(ctx, codec)) } -pub fn try_decode_higher_order_udf(udf: &gen::HigherOrderUdf) -> Result> { +pub(super) fn try_decode_higher_order_udf( + udf: &r#gen::HigherOrderUdf, +) -> Result> { let udf_kind = udf .higher_order_udf_kind .as_ref() .cloned() .ok_or_else(|| plan_datafusion_err!("missing higher-order function UDF"))?; Ok(match udf_kind { - HigherOrderUdfKind::Filter(gen::SparkArrayFilterUdf { index_first }) => { + HigherOrderUdfKind::Filter(r#gen::SparkArrayFilterUdf { index_first }) => { if index_first { Arc::new(HigherOrderUDF::new_from_impl( SparkArrayFilter::new_index_first(), @@ -96,7 +118,7 @@ pub fn try_decode_higher_order_udf(udf: &gen::HigherOrderUdf) -> Result { + HigherOrderUdfKind::Transform(r#gen::SparkArrayTransformUdf { index_first }) => { if index_first { Arc::new(HigherOrderUDF::new_from_impl( SparkArrayTransform::new_index_first(), @@ -105,7 +127,7 @@ pub fn try_decode_higher_order_udf(udf: &gen::HigherOrderUdf) -> Result { + HigherOrderUdfKind::Aggregate(r#gen::SparkArrayAggregateUdf { element_first }) => { if element_first { Arc::new(HigherOrderUDF::new_from_impl( SparkArrayAggregate::new_element_first(), @@ -114,13 +136,13 @@ pub fn try_decode_higher_order_udf(udf: &gen::HigherOrderUdf) -> Result { + HigherOrderUdfKind::Exists(r#gen::SparkArrayExistsUdf {}) => { Arc::new(HigherOrderUDF::new_from_impl(SparkArrayExists::new())) } - HigherOrderUdfKind::Forall(gen::SparkArrayForallUdf {}) => { + HigherOrderUdfKind::Forall(r#gen::SparkArrayForallUdf {}) => { Arc::new(HigherOrderUDF::new_from_impl(SparkArrayForall::new())) } - HigherOrderUdfKind::Sort(gen::SparkArraySortUdf { swapped }) => { + HigherOrderUdfKind::Sort(r#gen::SparkArraySortUdf { swapped }) => { if swapped { Arc::new(HigherOrderUDF::new_from_impl(SparkArraySort::new_swapped())) } else { diff --git a/crates/sail-execution/src/proto/encode.rs b/crates/sail-execution/src/proto/encode.rs index 5af3d6ac9..ec5dea4c1 100644 --- a/crates/sail-execution/src/proto/encode.rs +++ b/crates/sail-execution/src/proto/encode.rs @@ -1,7 +1,9 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{FieldRef, Schema}; -use datafusion::common::{plan_err, Result}; +use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; +use datafusion::common::{Result, plan_err}; +use datafusion::datasource::source::DataSourceExec; use datafusion::physical_expr::{HigherOrderFunctionExpr, PhysicalExpr}; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::generated::datafusion_common as gen_datafusion_common; @@ -14,34 +16,60 @@ use sail_function::scalar::array::spark_array_filter::SparkArrayFilter; use sail_function::scalar::array::spark_array_forall::SparkArrayForall; use sail_function::scalar::array::spark_array_sort::SparkArraySort; use sail_function::scalar::array::spark_array_transform::SparkArrayTransform; +use sail_physical_plan::data_source::RemoteDataSourceExec; -use crate::plan::gen; -use crate::plan::gen::higher_order_udf::HigherOrderUdfKind; +use crate::plan::r#gen; +use crate::plan::r#gen::higher_order_udf::HigherOrderUdfKind; use crate::proto::converter::RemotePhysicalProtoConverter; -pub fn try_encode_message(message: M) -> Result> +pub fn encode_remote_physical_plan( + codec: &dyn PhysicalExtensionCodec, + plan: Arc, +) -> Result> { + let plan = plan + .transform(|node| { + if let Some(data_source) = node.downcast_ref::() { + let node = + Arc::new(RemoteDataSourceExec::new(data_source)) as Arc; + Ok(Transformed::yes(node)) + } else { + Ok(Transformed::no(node)) + } + }) + .data()?; + try_encode_physical_plan(codec, plan) +} + +pub fn encode_remote_physical_expr( + codec: &dyn PhysicalExtensionCodec, + expr: &Arc, +) -> Result> { + try_encode_physical_expr(codec, expr) +} + +pub(super) fn try_encode_message(message: M) -> Result> where M: Message, { Ok(message.encode_to_vec()) } -pub fn try_encode_schema(schema: &Schema) -> Result> { +pub(super) fn try_encode_schema(schema: &Schema) -> Result> { try_encode_message::(schema.try_into()?) } -pub fn try_encode_field_ref(field: &FieldRef) -> Result> { +pub(super) fn try_encode_field_ref(field: &FieldRef) -> Result> { try_encode_message::(field.as_ref().try_into()?) } -pub fn try_encode_physical_plan( +pub(super) fn try_encode_physical_plan( codec: &dyn PhysicalExtensionCodec, plan: Arc, ) -> Result> { try_encode_message(physical_plan_to_proto(codec, plan)?) } -pub fn physical_plan_to_proto( +pub(super) fn physical_plan_to_proto( codec: &dyn PhysicalExtensionCodec, plan: Arc, ) -> Result { @@ -52,14 +80,14 @@ pub fn physical_plan_to_proto( ) } -pub fn try_encode_physical_expr( +pub(super) fn try_encode_physical_expr( codec: &dyn PhysicalExtensionCodec, expr: &Arc, ) -> Result> { try_encode_message(physical_expr_to_proto(codec, expr)?) } -pub fn physical_expr_to_proto( +pub(super) fn physical_expr_to_proto( codec: &dyn PhysicalExtensionCodec, expr: &Arc, ) -> Result { @@ -67,32 +95,34 @@ pub fn physical_expr_to_proto( converter.physical_expr_to_proto(expr, codec) } -pub fn try_encode_higher_order_udf(hof: &HigherOrderFunctionExpr) -> Result { +pub(super) fn try_encode_higher_order_udf( + hof: &HigherOrderFunctionExpr, +) -> Result { let udf_inner = hof.fun().inner().as_ref() as &dyn std::any::Any; let udf_kind = if let Some(filter) = udf_inner.downcast_ref::() { - HigherOrderUdfKind::Filter(gen::SparkArrayFilterUdf { + HigherOrderUdfKind::Filter(r#gen::SparkArrayFilterUdf { index_first: filter.is_index_first(), }) } else if let Some(transform) = udf_inner.downcast_ref::() { - HigherOrderUdfKind::Transform(gen::SparkArrayTransformUdf { + HigherOrderUdfKind::Transform(r#gen::SparkArrayTransformUdf { index_first: transform.is_index_first(), }) } else if let Some(aggregate) = udf_inner.downcast_ref::() { - HigherOrderUdfKind::Aggregate(gen::SparkArrayAggregateUdf { + HigherOrderUdfKind::Aggregate(r#gen::SparkArrayAggregateUdf { element_first: aggregate.is_element_first(), }) } else if udf_inner.is::() { - HigherOrderUdfKind::Exists(gen::SparkArrayExistsUdf {}) + HigherOrderUdfKind::Exists(r#gen::SparkArrayExistsUdf {}) } else if udf_inner.is::() { - HigherOrderUdfKind::Forall(gen::SparkArrayForallUdf {}) + HigherOrderUdfKind::Forall(r#gen::SparkArrayForallUdf {}) } else if let Some(sort) = udf_inner.downcast_ref::() { - HigherOrderUdfKind::Sort(gen::SparkArraySortUdf { + HigherOrderUdfKind::Sort(r#gen::SparkArraySortUdf { swapped: sort.is_swapped(), }) } else { return plan_err!("unsupported higher-order function: {}", hof.name()); }; - Ok(gen::HigherOrderUdf { + Ok(r#gen::HigherOrderUdf { higher_order_udf_kind: Some(udf_kind), }) } diff --git a/crates/sail-execution/src/proto/mod.rs b/crates/sail-execution/src/proto/mod.rs index 2810af76f..c40bdeae7 100644 --- a/crates/sail-execution/src/proto/mod.rs +++ b/crates/sail-execution/src/proto/mod.rs @@ -1,4 +1,8 @@ -pub mod codec; -pub mod converter; -pub mod decode; -pub mod encode; +mod codec; +mod converter; +mod decode; +mod encode; + +pub use codec::RemoteExecutionCodec; +pub use decode::{decode_remote_physical_expr, decode_remote_physical_plan}; +pub use encode::{encode_remote_physical_expr, encode_remote_physical_plan}; diff --git a/crates/sail-execution/src/rpc.rs b/crates/sail-execution/src/rpc.rs index 8372efa87..2056b3c34 100644 --- a/crates/sail-execution/src/rpc.rs +++ b/crates/sail-execution/src/rpc.rs @@ -2,8 +2,8 @@ use std::future::Future; use std::sync::Arc; use arrow_flight::flight_service_client::FlightServiceClient; -use sail_telemetry::layers::{TracingClientLayer, TracingClientService}; -use tokio::sync::{oneshot, OnceCell}; +use sail_common::telemetry::{TracingClientLayer, TracingClientService}; +use tokio::sync::{OnceCell, oneshot}; use tokio::task::JoinHandle; use tonic::transport::Channel; use tower::ServiceBuilder; diff --git a/crates/sail-execution/src/shuffle.rs b/crates/sail-execution/src/shuffle.rs new file mode 100644 index 000000000..9bd28636c --- /dev/null +++ b/crates/sail-execution/src/shuffle.rs @@ -0,0 +1,39 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShuffleBackendKind { + Flight, + Storage { + path: Option, + max_file_size: usize, + compression: ShuffleCompression, + }, +} + +impl From<&sail_common::config::ShuffleBackend> for ShuffleBackendKind { + fn from(value: &sail_common::config::ShuffleBackend) -> Self { + match value { + sail_common::config::ShuffleBackend::Flight => Self::Flight, + sail_common::config::ShuffleBackend::Storage(storage) => Self::Storage { + path: storage.path.clone(), + max_file_size: storage.max_file_size, + compression: storage.compression.clone().into(), + }, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ShuffleCompression { + None, + Lz4, + Zstd, +} + +impl From for ShuffleCompression { + fn from(value: sail_common::config::ShuffleCompression) -> Self { + match value { + sail_common::config::ShuffleCompression::None => Self::None, + sail_common::config::ShuffleCompression::Lz4 => Self::Lz4, + sail_common::config::ShuffleCompression::Zstd => Self::Zstd, + } + } +} diff --git a/crates/sail-execution/src/stream/error.rs b/crates/sail-execution/src/stream/error.rs index d94223143..f911e98b3 100644 --- a/crates/sail-execution/src/stream/error.rs +++ b/crates/sail-execution/src/stream/error.rs @@ -4,7 +4,7 @@ use std::fmt::Formatter; use std::sync::Arc; use arrow_flight::error::FlightError; -use sail_common_datafusion::error::{CommonErrorCause, RemotePythonError}; +use sail_common_datafusion::error::{CommonErrorCause, RemoteError}; use sail_python_udf::error::PyErrExtractor; use tonic::{Code, Status}; use tonic_types::{ErrorDetails, StatusExt}; @@ -39,16 +39,15 @@ impl From for TaskStreamError { fn from(status: Status) -> Self { if status.code() == Code::Internal { let details = status.get_error_details(); - if let Some(info) = details.error_info() { - if let ("task stream", "sail", Some(cause)) = ( + if let Some(info) = details.error_info() + && let ("task stream", "sail", Some(cause)) = ( info.reason.as_str(), info.domain.as_str(), info.metadata.get("cause"), - ) { - if let Ok(cause) = serde_json::from_str::(cause) { - return cause.into(); - } - } + ) + && let Ok(cause) = serde_json::from_str::(cause) + { + return cause.into(); } } Self::Unknown(status.message().to_string()) @@ -84,36 +83,7 @@ impl From for TaskStreamError { impl From for TaskStreamError { fn from(value: CommonErrorCause) -> Self { - match value { - CommonErrorCause::Unknown(x) - | CommonErrorCause::Internal(x) - | CommonErrorCause::NotImplemented(x) - | CommonErrorCause::InvalidArgument(x) - | CommonErrorCause::Io(x) - | CommonErrorCause::ArrowCast(x) - | CommonErrorCause::ArrowMemory(x) - | CommonErrorCause::ArrowParse(x) - | CommonErrorCause::ArrowCompute(x) - | CommonErrorCause::ArrowIpc(x) - | CommonErrorCause::ArrowCDataInterface(x) - | CommonErrorCause::ArrowDivideByZero(x) - | CommonErrorCause::ArrowArithmeticOverflow(x) - | CommonErrorCause::ArrowDictionaryKeyOverflow(x) - | CommonErrorCause::ArrowRunEndIndexOverflow(x) - | CommonErrorCause::ArrowOffsetOverflow(x) - | CommonErrorCause::FormatCsv(x) - | CommonErrorCause::FormatJson(x) - | CommonErrorCause::FormatParquet(x) - | CommonErrorCause::FormatAvro(x) - | CommonErrorCause::Plan(x) - | CommonErrorCause::Schema(x) - | CommonErrorCause::Configuration(x) - | CommonErrorCause::Execution(x) - | CommonErrorCause::DeltaTable(x) => Self::Unknown(x), - CommonErrorCause::Python(cause) => { - Self::External(Arc::new(RemotePythonError::from(cause))) - } - } + Self::External(Arc::new(RemoteError { cause: value })) } } @@ -121,7 +91,7 @@ impl From for CommonErrorCause { fn from(value: TaskStreamError) -> Self { match value { TaskStreamError::Unknown(x) => CommonErrorCause::Unknown(x), - TaskStreamError::External(e) => CommonErrorCause::new::(&e), + TaskStreamError::External(e) => CommonErrorCause::new::(e.as_ref()), } } } diff --git a/crates/sail-execution/src/stream/merge.rs b/crates/sail-execution/src/stream/merge.rs index f04e702e4..b245dd7f0 100644 --- a/crates/sail-execution/src/stream/merge.rs +++ b/crates/sail-execution/src/stream/merge.rs @@ -6,8 +6,8 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::Result; use datafusion::error::DataFusionError; use datafusion::execution::RecordBatchStream; -use futures::stream::{select_all, SelectAll}; use futures::Stream; +use futures::stream::{SelectAll, select_all}; use crate::stream::reader::TaskStreamSource; diff --git a/crates/sail-execution/src/stream/mod.rs b/crates/sail-execution/src/stream/mod.rs index 1bba67149..d79b447d6 100644 --- a/crates/sail-execution/src/stream/mod.rs +++ b/crates/sail-execution/src/stream/mod.rs @@ -3,6 +3,6 @@ pub mod merge; pub mod reader; pub mod writer; -pub mod gen { +pub mod r#gen { tonic::include_proto!("sail.stream"); } diff --git a/crates/sail-execution/src/stream/reader.rs b/crates/sail-execution/src/stream/reader.rs index e762c0c63..f77ed3858 100644 --- a/crates/sail-execution/src/stream/reader.rs +++ b/crates/sail-execution/src/stream/reader.rs @@ -18,7 +18,6 @@ pub enum TaskReadLocation { key: TaskStreamKey, }, Remote { - uri: String, key: TaskStreamKey, }, } @@ -32,8 +31,8 @@ impl fmt::Display for TaskReadLocation { TaskReadLocation::Worker { worker_id, key } => { write!(f, "Worker({worker_id}, {})", TaskStreamKeyDenseDisplay(key)) } - TaskReadLocation::Remote { uri, key } => { - write!(f, "Remote({uri}, {})", TaskStreamKeyDenseDisplay(key)) + TaskReadLocation::Remote { key } => { + write!(f, "Remote({})", TaskStreamKeyDenseDisplay(key)) } } } diff --git a/crates/sail-execution/src/stream/writer.rs b/crates/sail-execution/src/stream/writer.rs index a751d29cd..57217e1af 100644 --- a/crates/sail-execution/src/stream/writer.rs +++ b/crates/sail-execution/src/stream/writer.rs @@ -14,7 +14,6 @@ pub enum TaskWriteLocation { key: TaskStreamKey, }, Remote { - uri: String, key: TaskStreamKey, }, } @@ -34,8 +33,8 @@ impl fmt::Display for TaskWriteLocation { TaskWriteLocation::Local { key, storage } => { write!(f, "Local({storage}, {})", TaskStreamKeyDenseDisplay(key)) } - TaskWriteLocation::Remote { uri, key } => { - write!(f, "Remote({uri}, {})", TaskStreamKeyDenseDisplay(key)) + TaskWriteLocation::Remote { key } => { + write!(f, "Remote({})", TaskStreamKeyDenseDisplay(key)) } } } @@ -78,7 +77,6 @@ pub enum TaskStreamSinkState { /// The sink is ready to accept more writes. Ok, /// The sink has encountered an error and no further writes should be attempted. - #[expect(unused)] Error(DataFusionError), /// The sink has been closed and no further writes should be attempted. Closed, diff --git a/crates/sail-execution/src/stream_accessor/core.rs b/crates/sail-execution/src/stream_accessor/core.rs index f9077cf8d..2ec759e5b 100644 --- a/crates/sail-execution/src/stream_accessor/core.rs +++ b/crates/sail-execution/src/stream_accessor/core.rs @@ -1,6 +1,9 @@ +use std::sync::Arc; + use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{DataFusionError, Result}; -use sail_server::actor::{Actor, ActorHandle}; +use datafusion::execution::TaskContext; +use sail_common::actor::{Actor, ActorHandle}; use tokio::sync::oneshot; use crate::stream::reader::{TaskReadLocation, TaskStreamReader, TaskStreamSource}; @@ -8,8 +11,8 @@ use crate::stream::writer::{TaskStreamSink, TaskStreamWriter, TaskWriteLocation} use crate::stream_accessor::{StreamAccessor, StreamAccessorMessage}; impl StreamAccessor { - pub fn new(handle: ActorHandle) -> Self { - Self { handle } + pub fn new(handle: ActorHandle, context: Arc) -> Self { + Self { handle, context } } } @@ -31,8 +34,8 @@ where TaskReadLocation::Worker { worker_id, key } => { T::Message::fetch_worker_stream(*worker_id, key.clone(), schema, tx) } - TaskReadLocation::Remote { uri, key } => { - T::Message::fetch_remote_stream(uri.clone(), key.clone(), schema, tx) + TaskReadLocation::Remote { key } => { + T::Message::fetch_remote_stream(key.clone(), schema, self.context.clone(), tx) } }; self.handle.send(event).await.map_err(|_| { @@ -59,8 +62,8 @@ where TaskWriteLocation::Local { key, storage } => { T::Message::create_local_stream(key.clone(), *storage, schema, tx) } - TaskWriteLocation::Remote { uri, key } => { - T::Message::create_remote_stream(uri.clone(), key.clone(), schema, tx) + TaskWriteLocation::Remote { key } => { + T::Message::create_remote_stream(key.clone(), schema, self.context.clone(), tx) } }; self.handle.send(event).await.map_err(|_| { diff --git a/crates/sail-execution/src/stream_accessor/mod.rs b/crates/sail-execution/src/stream_accessor/mod.rs index 4336d44a7..679363a28 100644 --- a/crates/sail-execution/src/stream_accessor/mod.rs +++ b/crates/sail-execution/src/stream_accessor/mod.rs @@ -1,9 +1,11 @@ mod core; use std::fmt; +use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; -use sail_server::actor::{Actor, ActorHandle}; +use datafusion::execution::TaskContext; +use sail_common::actor::{Actor, ActorHandle}; use tokio::sync::oneshot; use crate::driver::DriverEvent; @@ -15,6 +17,16 @@ use crate::worker::{WorkerEvent, WorkerStreamOwner}; pub struct StreamAccessor { handle: ActorHandle, + context: Arc, +} + +impl Clone for StreamAccessor { + fn clone(&self) -> Self { + Self { + handle: self.handle.clone(), + context: Arc::clone(&self.context), + } + } } impl fmt::Debug for StreamAccessor { @@ -32,9 +44,9 @@ pub trait StreamAccessorMessage { ) -> Self; fn create_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> Self; @@ -52,9 +64,9 @@ pub trait StreamAccessorMessage { ) -> Self; fn fetch_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> Self; } @@ -75,15 +87,15 @@ impl StreamAccessorMessage for DriverEvent { } fn create_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> Self { DriverEvent::CreateRemoteStream { - uri, key, schema, + context, result, } } @@ -111,15 +123,15 @@ impl StreamAccessorMessage for DriverEvent { } fn fetch_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> Self { DriverEvent::FetchRemoteStream { - uri, key, schema, + context, result, } } @@ -141,15 +153,15 @@ impl StreamAccessorMessage for WorkerEvent { } fn create_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> Self { WorkerEvent::CreateRemoteStream { - uri, key, schema, + context, result, } } @@ -180,15 +192,15 @@ impl StreamAccessorMessage for WorkerEvent { } fn fetch_remote_stream( - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> Self { WorkerEvent::FetchRemoteStream { - uri, key, schema, + context, result, } } diff --git a/crates/sail-execution/src/stream_manager/core.rs b/crates/sail-execution/src/stream_manager/core.rs index 8482aabbf..3a7888b3e 100644 --- a/crates/sail-execution/src/stream_manager/core.rs +++ b/crates/sail-execution/src/stream_manager/core.rs @@ -1,12 +1,14 @@ -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::Arc; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use log::warn; +use sail_common::actor::{Actor, ActorContext}; use sail_common_datafusion::error::CommonErrorCause; use sail_python_udf::error::PyErrExtractor; -use sail_server::actor::{Actor, ActorContext}; use tokio::sync::mpsc; use tonic::codegen::tokio_stream::wrappers::ReceiverStream; @@ -17,12 +19,27 @@ use crate::stream::reader::TaskStreamSource; use crate::stream::writer::{LocalStreamStorage, TaskStreamSink}; use crate::stream_manager::local::{LocalStream, MemoryStream}; use crate::stream_manager::options::StreamManagerOptions; +use crate::stream_manager::remote::RemoteStreamManager; use crate::stream_manager::{LocalStreamState, StreamManager, StreamManagerMessage}; impl StreamManager { pub fn new(options: StreamManagerOptions) -> Self { + let remote_streams = match &options.shuffle_backend { + crate::shuffle::ShuffleBackendKind::Flight => None, + crate::shuffle::ShuffleBackendKind::Storage { + path, + max_file_size, + compression, + } => Some(Arc::new(RemoteStreamManager::new( + path.clone(), + options.session_id.clone(), + *max_file_size, + *compression, + ))), + }; Self { options, + remote_streams, local_streams: HashMap::new(), } } @@ -87,13 +104,16 @@ impl StreamManager { pub fn create_remote_stream( &mut self, - _uri: String, - _key: TaskStreamKey, - _schema: SchemaRef, + key: TaskStreamKey, + schema: SchemaRef, + context: &TaskContext, ) -> ExecutionResult> { - Err(ExecutionError::InternalError( - "not implemented: remote stream".to_string(), - )) + let Some(remote_streams) = &self.remote_streams else { + return Err(ExecutionError::InternalError( + "remote stream requested without a storage shuffle backend".to_string(), + )); + }; + remote_streams.create_stream(key, schema, context) } pub fn fetch_local_stream( @@ -135,17 +155,20 @@ impl StreamManager { pub fn fetch_remote_stream( &mut self, _ctx: &mut ActorContext, - _uri: String, - _key: &TaskStreamKey, - _schema: SchemaRef, + key: &TaskStreamKey, + schema: SchemaRef, + context: &TaskContext, ) -> ExecutionResult where T: Actor, T::Message: StreamManagerMessage, { - Err(ExecutionError::InternalError( - "not implemented: fetch remote stream".to_string(), - )) + let Some(remote_streams) = &self.remote_streams else { + return Err(ExecutionError::InternalError( + "remote stream requested without a storage shuffle backend".to_string(), + )); + }; + remote_streams.fetch_stream(key.clone(), schema, context) } pub fn remove_local_streams(&mut self, job_id: JobId, stage: Option) { @@ -159,13 +182,21 @@ impl StreamManager { pub fn remove_remote_streams( &mut self, - _ctx: &mut ActorContext, - _job_id: JobId, - _stage: Option, + ctx: &mut ActorContext, + job_id: JobId, + stage: Option, + context: Arc, ) where T: Actor, { - warn!("removing remote streams is not implemented"); + let Some(remote_streams) = self.remote_streams.clone() else { + return; + }; + ctx.spawn(async move { + if let Err(e) = remote_streams.remove_streams(job_id, stage, &context).await { + warn!("failed to remove remote shuffle data for job {job_id}: {e}"); + } + }); } pub fn fail_local_stream_if_pending(&mut self, key: &TaskStreamKey) { diff --git a/crates/sail-execution/src/stream_manager/mod.rs b/crates/sail-execution/src/stream_manager/mod.rs index a06b961cf..f03c652a2 100644 --- a/crates/sail-execution/src/stream_manager/mod.rs +++ b/crates/sail-execution/src/stream_manager/mod.rs @@ -1,8 +1,10 @@ mod core; mod local; mod options; +mod remote; use std::collections::HashMap; +use std::sync::Arc; use datafusion::arrow::array::RecordBatch; pub use options::StreamManagerOptions; @@ -16,6 +18,7 @@ use crate::worker::WorkerEvent; pub struct StreamManager { options: StreamManagerOptions, + remote_streams: Option>, local_streams: HashMap, } diff --git a/crates/sail-execution/src/stream_manager/options.rs b/crates/sail-execution/src/stream_manager/options.rs index e50d6bde1..284b8b515 100644 --- a/crates/sail-execution/src/stream_manager/options.rs +++ b/crates/sail-execution/src/stream_manager/options.rs @@ -1,19 +1,24 @@ use std::time::Duration; use crate::driver::DriverOptions; +use crate::shuffle::ShuffleBackendKind; use crate::worker::WorkerOptions; #[readonly::make] pub struct StreamManagerOptions { + pub session_id: String, pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, + pub shuffle_backend: ShuffleBackendKind, } impl From<&DriverOptions> for StreamManagerOptions { fn from(options: &DriverOptions) -> Self { Self { + session_id: options.session_id.clone(), task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, + shuffle_backend: options.shuffle_backend.clone(), } } } @@ -21,8 +26,10 @@ impl From<&DriverOptions> for StreamManagerOptions { impl From<&WorkerOptions> for StreamManagerOptions { fn from(options: &WorkerOptions) -> Self { Self { + session_id: options.session_id.clone(), task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, + shuffle_backend: options.shuffle_backend.clone(), } } } diff --git a/crates/sail-execution/src/stream_manager/remote.rs b/crates/sail-execution/src/stream_manager/remote.rs new file mode 100644 index 000000000..f645367c2 --- /dev/null +++ b/crates/sail-execution/src/stream_manager/remote.rs @@ -0,0 +1,388 @@ +use std::io::Cursor; +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::arrow::error::ArrowError; +use datafusion::arrow::ipc::CompressionType; +use datafusion::arrow::ipc::reader::StreamReader; +use datafusion::arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use datafusion::common::{DataFusionError, Result}; +use datafusion::execution::TaskContext; +use futures::{StreamExt, TryStreamExt}; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; +use url::Url; + +use crate::error::{ExecutionError, ExecutionResult}; +use crate::id::{JobId, TaskStreamKey}; +use crate::shuffle::ShuffleCompression; +use crate::stream::error::{TaskStreamError, TaskStreamResult}; +use crate::stream::reader::TaskStreamSource; +use crate::stream::writer::{TaskStreamSink, TaskStreamSinkState}; + +/// Object-storage implementation for a blocking shuffle stream. Every task attempt owns a +/// directory, allowing retries to be selected by the task scheduler without overwriting an +/// earlier attempt. +pub(super) struct RemoteStreamManager { + path: Option, + session_id: String, + max_file_size: usize, + compression: ShuffleCompression, +} + +impl RemoteStreamManager { + pub(super) fn new( + path: Option, + session_id: String, + max_file_size: usize, + compression: ShuffleCompression, + ) -> Self { + Self { + path, + session_id, + max_file_size, + compression, + } + } + + pub(super) fn create_stream( + &self, + key: TaskStreamKey, + schema: SchemaRef, + context: &TaskContext, + ) -> ExecutionResult> { + if self.max_file_size == 0 { + return Err(ExecutionError::InvalidArgument( + "cluster.shuffle_backend.storage.max_file_size must be greater than zero" + .to_string(), + )); + } + let (store, prefix) = self.store_and_prefix(&key, context)?; + let options = IpcWriteOptions::default() + .try_with_compression(match self.compression { + ShuffleCompression::None => None, + ShuffleCompression::Lz4 => Some(CompressionType::LZ4_FRAME), + ShuffleCompression::Zstd => Some(CompressionType::ZSTD), + }) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(Box::new(RemoteStreamSink::new( + store, + prefix, + schema, + options, + self.max_file_size, + )?)) + } + + pub(super) fn fetch_stream( + &self, + key: TaskStreamKey, + _schema: SchemaRef, + context: &TaskContext, + ) -> ExecutionResult { + let (store, prefix) = self.store_and_prefix(&key, context)?; + let list_store = Arc::clone(&store); + let output = futures::stream::once(async move { + let mut locations = list_store + .list(Some(&prefix)) + .map_err(|e| TaskStreamError::External(Arc::new(e))) + .try_collect::>() + .await? + .into_iter() + .map(|meta| meta.location) + .collect::>(); + locations.sort(); + Ok::<_, TaskStreamError>(locations) + }) + .flat_map(move |result| -> TaskStreamSource { + match result { + Ok(locations) => Box::pin(futures::stream::try_unfold( + RemoteReadState { + store: Arc::clone(&store), + locations: locations.into_iter(), + reader: None, + }, + |mut state| async move { + loop { + if let Some(reader) = state.reader.as_mut() { + match reader.next() { + Some(Ok(batch)) => return Ok(Some((batch, state))), + Some(Err(error)) => { + return Err(TaskStreamError::External(Arc::new(error))); + } + None => state.reader = None, + } + } + let Some(location) = state.locations.next() else { + return Ok(None); + }; + let bytes = state + .store + .get(&location) + .await + .map_err(|e| TaskStreamError::External(Arc::new(e)))? + .bytes() + .await + .map_err(|e| TaskStreamError::External(Arc::new(e)))?; + let reader = StreamReader::try_new(Cursor::new(bytes), None) + .map_err(|e| TaskStreamError::External(Arc::new(e)))?; + state.reader = Some(Box::new(reader)); + } + }, + )), + Err(error) => Box::pin(futures::stream::iter(vec![Err(error)])), + } + }); + Ok(Box::pin(output)) + } + + pub(super) async fn remove_streams( + &self, + job_id: JobId, + stage: Option, + context: &TaskContext, + ) -> Result<()> { + // All shuffle data for a job is beneath this prefix, so terminal job cleanup removes + // every attempt. Stage cleanup can safely reclaim data once all of its consumers finish. + let Some((store, prefix)) = self + .store_and_cleanup_prefix(job_id, stage, context) + .map_err(|e| DataFusionError::External(Box::new(e)))? + else { + return Ok(()); + }; + let locations = store + .list(Some(&prefix)) + .map_ok(|meta| meta.location) + .boxed(); + store + .delete_stream(locations) + .try_collect::>() + .await + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(()) + } + + fn store_and_prefix( + &self, + key: &TaskStreamKey, + context: &TaskContext, + ) -> ExecutionResult<(Arc, Path)> { + let path = self.path.as_deref().ok_or_else(|| { + ExecutionError::InvalidArgument("missing shuffle storage path".to_string()) + })?; + let url = Url::parse(path).map_err(|e| ExecutionError::InvalidArgument(e.to_string()))?; + let store = context + .runtime_env() + .object_store_registry + .get_store(&url)?; + Ok((store, stream_prefix(&url, key, &self.session_id)?)) + } + + fn store_and_cleanup_prefix( + &self, + job_id: JobId, + stage: Option, + context: &TaskContext, + ) -> ExecutionResult, Path)>> { + let Some(path) = self.path.as_deref() else { + return Ok(None); + }; + let url = Url::parse(path).map_err(|e| ExecutionError::InvalidArgument(e.to_string()))?; + let store = context + .runtime_env() + .object_store_registry + .get_store(&url)?; + Ok(Some(( + store, + shuffle_prefix(&url, &self.session_id, job_id, stage)?, + ))) + } +} + +struct RemoteReadState { + store: Arc, + locations: std::vec::IntoIter, + reader: Option, +} + +type RemoteBatchReader = + Box> + Send>; + +struct RemoteStreamSink { + store: Arc, + prefix: Path, + schema: SchemaRef, + options: IpcWriteOptions, + max_file_size: usize, + writer: Option>>, + file_index: usize, + has_batches: bool, +} + +impl RemoteStreamSink { + fn new( + store: Arc, + prefix: Path, + schema: SchemaRef, + options: IpcWriteOptions, + max_file_size: usize, + ) -> Result { + let writer = + StreamWriter::try_new_with_options(Vec::new(), schema.as_ref(), options.clone()) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(Self { + store, + prefix, + schema, + options, + max_file_size, + writer: Some(writer), + file_index: 0, + has_batches: false, + }) + } + + fn new_writer(&self) -> Result>> { + StreamWriter::try_new_with_options(Vec::new(), self.schema.as_ref(), self.options.clone()) + .map_err(|e| DataFusionError::External(Box::new(e))) + } + + async fn flush_file(&mut self) -> Result<()> { + if !self.has_batches { + return Ok(()); + } + let mut writer = self.writer.take().ok_or_else(|| { + DataFusionError::Internal("remote shuffle writer is not present".to_string()) + })?; + writer + .finish() + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let bytes = writer + .into_inner() + .map_err(|e| DataFusionError::External(Box::new(e)))?; + let path = self + .prefix + .clone() + .join(format!("part-{:020}.arrow", self.file_index)); + self.store + .put(&path, PutPayload::from(bytes)) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))?; + self.file_index += 1; + self.has_batches = false; + self.writer = Some(self.new_writer()?); + Ok(()) + } +} + +#[tonic::async_trait] +impl TaskStreamSink for RemoteStreamSink { + async fn write(&mut self, batch: TaskStreamResult) -> TaskStreamSinkState { + let batch = match batch { + Ok(batch) => batch, + Err(error) => { + return TaskStreamSinkState::Error(DataFusionError::External(Box::new(error))); + } + }; + let result = (|| -> Result { + let writer = self.writer.as_mut().ok_or_else(|| { + DataFusionError::Internal("remote shuffle writer is not present".to_string()) + })?; + writer + .write(&batch) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + self.has_batches = true; + Ok(writer.get_ref().len() >= self.max_file_size) + })(); + match result { + Ok(true) => match self.flush_file().await { + Ok(()) => TaskStreamSinkState::Ok, + Err(e) => TaskStreamSinkState::Error(e), + }, + Ok(false) => TaskStreamSinkState::Ok, + Err(e) => TaskStreamSinkState::Error(e), + } + } + + async fn close(mut self: Box) -> Result<()> { + self.flush_file().await + } +} + +fn stream_prefix(url: &Url, key: &TaskStreamKey, session_id: &str) -> ExecutionResult { + Ok( + shuffle_prefix(url, session_id, key.job_id, Some(key.stage))? + .join(format!("partition-{}", key.partition)) + .join(format!("attempt-{}", key.attempt)) + .join(format!("channel-{}", key.channel)), + ) +} + +fn shuffle_prefix( + url: &Url, + session_id: &str, + job_id: JobId, + stage: Option, +) -> ExecutionResult { + let mut prefix = Path::from_url_path(url.path()) + .map_err(|e| ExecutionError::InvalidArgument(e.to_string()))? + .join(session_id) + .join(format!("job-{job_id}")); + if let Some(stage) = stage { + prefix = prefix.join(format!("stage-{stage}")); + } + Ok(prefix) +} + +#[expect(clippy::unwrap_used)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shuffle_prefix_decodes_url_path() { + let url = Url::parse("file:///tmp/sail%20shuffle").unwrap(); + assert_eq!( + shuffle_prefix(&url, "session-1", JobId::from(1), Some(2)).unwrap(), + Path::from("tmp/sail shuffle/session-1/job-1/stage-2") + ); + } + + #[test] + fn stream_prefix_includes_channel() { + let url = Url::parse("memory:///shuffle").unwrap(); + let key = TaskStreamKey { + job_id: JobId::from(1), + stage: 2, + partition: 3, + attempt: 4, + channel: 5, + }; + let prefix = stream_prefix(&url, &key, "session-1").unwrap(); + assert_eq!( + prefix, + Path::from("shuffle/session-1/job-1/stage-2/partition-3/attempt-4/channel-5") + ); + } + + #[test] + fn missing_storage_path_is_invalid() { + let manager = + RemoteStreamManager::new(None, "session-1".to_string(), 1, ShuffleCompression::None); + let key = TaskStreamKey { + job_id: JobId::from(1), + stage: 2, + partition: 3, + attempt: 4, + channel: 5, + }; + let error = manager + .store_and_prefix(&key, &TaskContext::default()) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid argument: missing shuffle storage path" + ); + } +} diff --git a/crates/sail-execution/src/stream_service/client.rs b/crates/sail-execution/src/stream_service/client.rs index ef6a58963..84c5c3f32 100644 --- a/crates/sail-execution/src/stream_service/client.rs +++ b/crates/sail-execution/src/stream_service/client.rs @@ -1,34 +1,44 @@ +use std::sync::Arc; + use arrow_flight::decode::FlightRecordBatchStream; use arrow_flight::flight_service_client::FlightServiceClient; use datafusion::arrow::datatypes::SchemaRef; use futures::TryStreamExt; use prost::Message; +use sail_common_datafusion::array::record_batch::cast_record_batch_positionally; use crate::error::ExecutionResult; -use crate::id::TaskStreamKey; +use crate::id::{DriverId, TaskStreamKey}; use crate::rpc::{ClientHandle, ClientOptions, ClientService}; -use crate::stream::gen::TaskStreamTicket; +use crate::stream::error::TaskStreamError; +use crate::stream::r#gen::{DriverTaskStreamTicket, TaskStreamTicket}; use crate::stream::reader::TaskStreamSource; +#[derive(Clone, Copy, Debug)] +pub enum TaskStreamOwner { + Driver { driver_id: DriverId }, + Worker, +} + #[derive(Clone)] pub struct TaskStreamFlightClient { inner: ClientHandle>, + owner: TaskStreamOwner, } impl TaskStreamFlightClient { - pub fn new(options: ClientOptions) -> Self { + pub fn new(options: ClientOptions, owner: TaskStreamOwner) -> Self { Self { // TODO: share connection with driver/worker client inner: ClientHandle::new(options), + owner, } } pub async fn fetch_task_stream( &self, key: TaskStreamKey, - // The schema is unused for now since we have only in-memory streams. - // The schema may be needed if we have on-disk streams. - _schema: SchemaRef, + schema: SchemaRef, ) -> ExecutionResult { let ticket = TaskStreamTicket { job_id: key.job_id.into(), @@ -37,10 +47,15 @@ impl TaskStreamFlightClient { attempt: key.attempt as u64, channel: key.channel as u64, }; - let ticket = { - let mut buf = Vec::with_capacity(ticket.encoded_len()); - ticket.encode(&mut buf)?; - buf + let ticket = match self.owner { + TaskStreamOwner::Driver { driver_id } => { + let ticket = DriverTaskStreamTicket { + driver_id: driver_id.into(), + stream: Some(ticket), + }; + ticket.encode_to_vec() + } + TaskStreamOwner::Worker => ticket.encode_to_vec(), }; let request = arrow_flight::Ticket { ticket: ticket.into(), @@ -48,6 +63,18 @@ impl TaskStreamFlightClient { let response = self.inner.get().await?.do_get(request).await?; let stream = response.into_inner().map_err(|e| e.into()); let stream = FlightRecordBatchStream::new_from_flight_data(stream).map_err(|e| e.into()); + // The Flight data encoder may have issue with the `LargeList` data type, causing + // schema mismatch. As a workaround, here we cast the record batch to the expected schema. + // https://github.com/apache/arrow-rs/issues/10291 + let stream = stream.and_then(move |batch| { + let result = if batch.schema().as_ref() == schema.as_ref() { + Ok(batch) + } else { + cast_record_batch_positionally(batch, schema.clone()) + .map_err(|e| TaskStreamError::External(Arc::new(e))) + }; + futures::future::ready(result) + }); Ok(Box::pin(stream) as TaskStreamSource) } } diff --git a/crates/sail-execution/src/stream_service/mod.rs b/crates/sail-execution/src/stream_service/mod.rs index 74ae74091..ffb1a03a1 100644 --- a/crates/sail-execution/src/stream_service/mod.rs +++ b/crates/sail-execution/src/stream_service/mod.rs @@ -1,5 +1,5 @@ mod client; mod server; -pub use client::TaskStreamFlightClient; -pub use server::{TaskStreamFetcher, TaskStreamFlightServer}; +pub use client::{TaskStreamFlightClient, TaskStreamOwner}; +pub use server::{TaskStreamFetcher, TaskStreamFlightServer, TaskStreamKeyDecoder}; diff --git a/crates/sail-execution/src/stream_service/server.rs b/crates/sail-execution/src/stream_service/server.rs index 47fa193fd..edb2ceff3 100644 --- a/crates/sail-execution/src/stream_service/server.rs +++ b/crates/sail-execution/src/stream_service/server.rs @@ -1,4 +1,4 @@ -use std::io::Cursor; +use std::fmt::Debug; use std::pin::Pin; use arrow_flight::encode::FlightDataEncoderBuilder; @@ -12,28 +12,61 @@ use futures::{Stream, TryStreamExt}; use log::debug; use prost::Message; use tokio::sync::oneshot; -use tonic::{async_trait, Request, Response, Status, Streaming}; +use tonic::{Request, Response, Status, Streaming, async_trait}; use crate::error::ExecutionResult; use crate::id::TaskStreamKey; -use crate::stream::gen::TaskStreamTicket; +use crate::stream::r#gen::TaskStreamTicket; use crate::stream::reader::TaskStreamSource; +pub trait TaskStreamKeyDecoder: Debug + Send + 'static { + fn decode(bytes: &[u8]) -> Result + where + Self: Sized; +} + +impl From for TaskStreamKey { + fn from(ticket: TaskStreamTicket) -> Self { + let TaskStreamTicket { + job_id, + stage, + partition, + attempt, + channel, + } = ticket; + Self { + job_id: job_id.into(), + stage: stage as usize, + partition: partition as usize, + attempt: attempt as usize, + channel: channel as usize, + } + } +} + +impl TaskStreamKeyDecoder for TaskStreamKey { + fn decode(bytes: &[u8]) -> Result { + TaskStreamTicket::decode(bytes) + .map(Into::into) + .map_err(|e| Status::invalid_argument(e.to_string())) + } +} + #[async_trait] -pub trait TaskStreamFetcher: Send + Sync { +pub trait TaskStreamFetcher: Send + Sync { async fn fetch( &self, - key: TaskStreamKey, + key: K, sender: oneshot::Sender>, ) -> ExecutionResult<()>; } -pub struct TaskStreamFlightServer { - fetcher: Box, +pub struct TaskStreamFlightServer { + fetcher: Box>, } -impl TaskStreamFlightServer { - pub fn new(fetcher: Box) -> Self { +impl TaskStreamFlightServer { + pub fn new(fetcher: Box>) -> Self { Self { fetcher } } } @@ -41,7 +74,10 @@ impl TaskStreamFlightServer { type BoxedFlightStream = Pin> + Send + 'static>>; #[async_trait] -impl FlightService for TaskStreamFlightServer { +impl FlightService for TaskStreamFlightServer +where + K: TaskStreamKeyDecoder, +{ type HandshakeStream = BoxedFlightStream; async fn handshake( @@ -88,27 +124,9 @@ impl FlightService for TaskStreamFlightServer { request: Request, ) -> Result, Status> { let Ticket { ticket } = request.into_inner(); - let ticket = { - let mut buf = Cursor::new(&ticket); - TaskStreamTicket::decode(&mut buf) - .map_err(|e| Status::invalid_argument(e.to_string()))? - }; - debug!("{ticket:?}"); - let TaskStreamTicket { - job_id, - stage, - partition, - attempt, - channel, - } = ticket; + let key = K::decode(&ticket)?; + debug!("{key:?}"); let (tx, rx) = oneshot::channel(); - let key = TaskStreamKey { - job_id: job_id.into(), - stage: stage as usize, - partition: partition as usize, - attempt: attempt as usize, - channel: channel as usize, - }; self.fetcher.fetch(key, tx).await?; let stream = rx .await diff --git a/crates/sail-execution/src/task/definition.rs b/crates/sail-execution/src/task/definition.rs index d1465f984..eb0fa62d7 100644 --- a/crates/sail-execution/src/task/definition.rs +++ b/crates/sail-execution/src/task/definition.rs @@ -7,10 +7,10 @@ use datafusion_proto::physical_plan::PhysicalExtensionCodec; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::{JobId, TaskKey, TaskStreamKey, WorkerId}; -use crate::proto::decode::try_decode_physical_expr; +use crate::proto::decode_remote_physical_expr; use crate::stream::reader::TaskReadLocation; use crate::stream::writer::{LocalStreamStorage, TaskWriteLocation}; -use crate::task::gen; +use crate::task::r#gen; #[derive(Debug, Clone)] pub struct TaskDefinition { @@ -35,7 +35,6 @@ pub enum TaskInputLocator { keys: Vec>, }, Remote { - uri: String, stage: usize, keys: Vec>, }, @@ -71,17 +70,17 @@ pub enum TaskOutputDistribution { #[derive(Debug, Clone)] pub enum TaskOutputLocator { Local { replicas: usize }, - Remote { uri: String }, + Remote, } -impl From for gen::TaskDefinition { +impl From for r#gen::TaskDefinition { fn from(value: TaskDefinition) -> Self { let TaskDefinition { plan, inputs, output, } = value; - gen::TaskDefinition { + r#gen::TaskDefinition { plan: plan.to_vec(), inputs: inputs.into_iter().map(|x| x.into()).collect(), output: Some(output.into()), @@ -89,10 +88,10 @@ impl From for gen::TaskDefinition { } } -impl TryFrom for TaskDefinition { +impl TryFrom for TaskDefinition { type Error = ExecutionError; - fn try_from(value: gen::TaskDefinition) -> Result { + fn try_from(value: r#gen::TaskDefinition) -> Result { let inputs = value .inputs .into_iter() @@ -103,7 +102,7 @@ impl TryFrom for TaskDefinition { None => { return Err(ExecutionError::InvalidArgument( "cannot decode empty task output".to_string(), - )) + )); } }; Ok(TaskDefinition { @@ -114,64 +113,63 @@ impl TryFrom for TaskDefinition { } } -impl From for gen::TaskInput { +impl From for r#gen::TaskInput { fn from(value: TaskInput) -> Self { let TaskInput { locator } = value; - gen::TaskInput { + r#gen::TaskInput { locator: Some(locator.into()), } } } -impl TryFrom for TaskInput { +impl TryFrom for TaskInput { type Error = ExecutionError; - fn try_from(value: gen::TaskInput) -> Result { + fn try_from(value: r#gen::TaskInput) -> Result { let locator = match value.locator { Some(x) => x.try_into()?, None => { return Err(ExecutionError::InvalidArgument( "cannot decode empty task input locator".to_string(), - )) + )); } }; Ok(TaskInput { locator }) } } -impl From for gen::TaskInputLocator { +impl From for r#gen::TaskInputLocator { fn from(value: TaskInputLocator) -> Self { let kind = match value { TaskInputLocator::Driver { stage, keys } => { - gen::task_input_locator::Kind::Driver(gen::TaskInputDriverLocator { + r#gen::task_input_locator::Kind::Driver(r#gen::TaskInputDriverLocator { stage: stage as u64, keys: keys.into_iter().map(|x| x.into()).collect(), }) } TaskInputLocator::Worker { stage, keys } => { - gen::task_input_locator::Kind::Worker(gen::TaskInputWorkerLocator { + r#gen::task_input_locator::Kind::Worker(r#gen::TaskInputWorkerLocator { stage: stage as u64, keys: keys.into_iter().map(|x| x.into()).collect(), }) } - TaskInputLocator::Remote { uri, stage, keys } => { - gen::task_input_locator::Kind::Remote(gen::TaskInputRemoteLocator { - uri, + TaskInputLocator::Remote { stage, keys } => { + r#gen::task_input_locator::Kind::Remote(r#gen::TaskInputRemoteLocator { stage: stage as u64, keys: keys.into_iter().map(|x| x.into()).collect(), }) } }; - gen::TaskInputLocator { kind: Some(kind) } + r#gen::TaskInputLocator { kind: Some(kind) } } } -impl TryFrom for TaskInputLocator { +impl TryFrom for TaskInputLocator { type Error = ExecutionError; - fn try_from(value: gen::TaskInputLocator) -> Result { + fn try_from(value: r#gen::TaskInputLocator) -> Result { match value.kind { - Some(gen::task_input_locator::Kind::Driver(gen::TaskInputDriverLocator { + Some(r#gen::task_input_locator::Kind::Driver(r#gen::TaskInputDriverLocator { stage, keys, })) => { @@ -184,7 +182,7 @@ impl TryFrom for TaskInputLocator { keys, }) } - Some(gen::task_input_locator::Kind::Worker(gen::TaskInputWorkerLocator { + Some(r#gen::task_input_locator::Kind::Worker(r#gen::TaskInputWorkerLocator { stage, keys, })) => { @@ -197,8 +195,7 @@ impl TryFrom for TaskInputLocator { keys, }) } - Some(gen::task_input_locator::Kind::Remote(gen::TaskInputRemoteLocator { - uri, + Some(r#gen::task_input_locator::Kind::Remote(r#gen::TaskInputRemoteLocator { stage, keys, })) => { @@ -207,7 +204,6 @@ impl TryFrom for TaskInputLocator { .map(|x| x.try_into()) .collect::>>()?; Ok(TaskInputLocator::Remote { - uri, stage: stage as usize, keys, }) @@ -219,14 +215,14 @@ impl TryFrom for TaskInputLocator { } } -impl From for gen::TaskInputDriverKey { +impl From for r#gen::TaskInputDriverKey { fn from(value: TaskInputKey) -> Self { let TaskInputKey { partition, attempt, channel, } = value; - gen::TaskInputDriverKey { + r#gen::TaskInputDriverKey { partition: partition as u64, attempt: attempt as u64, channel: channel as u64, @@ -234,10 +230,10 @@ impl From for gen::TaskInputDriverKey { } } -impl TryFrom for TaskInputKey { +impl TryFrom for TaskInputKey { type Error = ExecutionError; - fn try_from(value: gen::TaskInputDriverKey) -> Result { + fn try_from(value: r#gen::TaskInputDriverKey) -> Result { Ok(TaskInputKey { partition: value.partition as usize, attempt: value.attempt as usize, @@ -246,18 +242,18 @@ impl TryFrom for TaskInputKey { } } -impl From> for gen::TaskInputDriverKeyList { +impl From> for r#gen::TaskInputDriverKeyList { fn from(value: Vec) -> Self { - gen::TaskInputDriverKeyList { + r#gen::TaskInputDriverKeyList { keys: value.into_iter().map(|x| x.into()).collect(), } } } -impl TryFrom for Vec { +impl TryFrom for Vec { type Error = ExecutionError; - fn try_from(value: gen::TaskInputDriverKeyList) -> Result { + fn try_from(value: r#gen::TaskInputDriverKeyList) -> Result { value .keys .into_iter() @@ -266,7 +262,7 @@ impl TryFrom for Vec { } } -impl From<(WorkerId, TaskInputKey)> for gen::TaskInputWorkerKey { +impl From<(WorkerId, TaskInputKey)> for r#gen::TaskInputWorkerKey { fn from(value: (WorkerId, TaskInputKey)) -> Self { let ( worker_id, @@ -276,7 +272,7 @@ impl From<(WorkerId, TaskInputKey)> for gen::TaskInputWorkerKey { channel, }, ) = value; - gen::TaskInputWorkerKey { + r#gen::TaskInputWorkerKey { worker_id: worker_id.into(), partition: partition as u64, attempt: attempt as u64, @@ -285,10 +281,10 @@ impl From<(WorkerId, TaskInputKey)> for gen::TaskInputWorkerKey { } } -impl TryFrom for (WorkerId, TaskInputKey) { +impl TryFrom for (WorkerId, TaskInputKey) { type Error = ExecutionError; - fn try_from(value: gen::TaskInputWorkerKey) -> Result { + fn try_from(value: r#gen::TaskInputWorkerKey) -> Result { Ok(( value.worker_id.into(), TaskInputKey { @@ -300,18 +296,18 @@ impl TryFrom for (WorkerId, TaskInputKey) { } } -impl From> for gen::TaskInputWorkerKeyList { +impl From> for r#gen::TaskInputWorkerKeyList { fn from(value: Vec<(WorkerId, TaskInputKey)>) -> Self { - gen::TaskInputWorkerKeyList { + r#gen::TaskInputWorkerKeyList { keys: value.into_iter().map(|x| x.into()).collect(), } } } -impl TryFrom for Vec<(WorkerId, TaskInputKey)> { +impl TryFrom for Vec<(WorkerId, TaskInputKey)> { type Error = ExecutionError; - fn try_from(value: gen::TaskInputWorkerKeyList) -> Result { + fn try_from(value: r#gen::TaskInputWorkerKeyList) -> Result { value .keys .into_iter() @@ -320,14 +316,14 @@ impl TryFrom for Vec<(WorkerId, TaskInputKey)> { } } -impl From for gen::TaskInputRemoteKey { +impl From for r#gen::TaskInputRemoteKey { fn from(value: TaskInputKey) -> Self { let TaskInputKey { partition, attempt, channel, } = value; - gen::TaskInputRemoteKey { + r#gen::TaskInputRemoteKey { partition: partition as u64, attempt: attempt as u64, channel: channel as u64, @@ -335,10 +331,10 @@ impl From for gen::TaskInputRemoteKey { } } -impl TryFrom for TaskInputKey { +impl TryFrom for TaskInputKey { type Error = ExecutionError; - fn try_from(value: gen::TaskInputRemoteKey) -> Result { + fn try_from(value: r#gen::TaskInputRemoteKey) -> Result { Ok(TaskInputKey { partition: value.partition as usize, attempt: value.attempt as usize, @@ -347,18 +343,18 @@ impl TryFrom for TaskInputKey { } } -impl From> for gen::TaskInputRemoteKeyList { +impl From> for r#gen::TaskInputRemoteKeyList { fn from(value: Vec) -> Self { - gen::TaskInputRemoteKeyList { + r#gen::TaskInputRemoteKeyList { keys: value.into_iter().map(|x| x.into()).collect(), } } } -impl TryFrom for Vec { +impl TryFrom for Vec { type Error = ExecutionError; - fn try_from(value: gen::TaskInputRemoteKeyList) -> Result { + fn try_from(value: r#gen::TaskInputRemoteKeyList) -> Result { value .keys .into_iter() @@ -367,29 +363,29 @@ impl TryFrom for Vec { } } -impl From for gen::TaskOutput { +impl From for r#gen::TaskOutput { fn from(value: TaskOutput) -> Self { let TaskOutput { distribution, locator, } = value; - gen::TaskOutput { + r#gen::TaskOutput { distribution: Some(distribution.into()), locator: Some(locator.into()), } } } -impl TryFrom for TaskOutput { +impl TryFrom for TaskOutput { type Error = ExecutionError; - fn try_from(value: gen::TaskOutput) -> Result { + fn try_from(value: r#gen::TaskOutput) -> Result { let distribution = match value.distribution { Some(x) => x.try_into()?, None => { return Err(ExecutionError::InvalidArgument( "cannot decode empty task output distribution".to_string(), - )) + )); } }; let locator = match value.locator { @@ -397,7 +393,7 @@ impl TryFrom for TaskOutput { None => { return Err(ExecutionError::InvalidArgument( "cannot decode empty task output locator".to_string(), - )) + )); } }; Ok(TaskOutput { @@ -407,53 +403,52 @@ impl TryFrom for TaskOutput { } } -impl From for gen::TaskOutputDistribution { +impl From for r#gen::TaskOutputDistribution { fn from(value: TaskOutputDistribution) -> Self { let kind = match value { TaskOutputDistribution::Hash { keys, channels } => { - gen::task_output_distribution::Kind::Hash(gen::TaskOutputHashDistribution { + r#gen::task_output_distribution::Kind::Hash(r#gen::TaskOutputHashDistribution { keys: keys.into_iter().map(|k| k.to_vec()).collect(), channels: channels as u64, }) } TaskOutputDistribution::RoundRobin { channels } => { - gen::task_output_distribution::Kind::RoundRobin( - gen::TaskOutputRoundRobinDistribution { + r#gen::task_output_distribution::Kind::RoundRobin( + r#gen::TaskOutputRoundRobinDistribution { channels: channels as u64, }, ) } TaskOutputDistribution::RoundRobinRow { channels } => { - gen::task_output_distribution::Kind::RoundRobinRow( - gen::TaskOutputRoundRobinRowDistribution { + r#gen::task_output_distribution::Kind::RoundRobinRow( + r#gen::TaskOutputRoundRobinRowDistribution { channels: channels as u64, }, ) } }; - gen::TaskOutputDistribution { kind: Some(kind) } + r#gen::TaskOutputDistribution { kind: Some(kind) } } } -impl TryFrom for TaskOutputDistribution { +impl TryFrom for TaskOutputDistribution { type Error = ExecutionError; - fn try_from(value: gen::TaskOutputDistribution) -> Result { + fn try_from(value: r#gen::TaskOutputDistribution) -> Result { match value.kind { - Some(gen::task_output_distribution::Kind::Hash(gen::TaskOutputHashDistribution { - keys, - channels, - })) => Ok(TaskOutputDistribution::Hash { + Some(r#gen::task_output_distribution::Kind::Hash( + r#gen::TaskOutputHashDistribution { keys, channels }, + )) => Ok(TaskOutputDistribution::Hash { keys: keys.into_iter().map(Arc::from).collect(), channels: channels as usize, }), - Some(gen::task_output_distribution::Kind::RoundRobin( - gen::TaskOutputRoundRobinDistribution { channels }, + Some(r#gen::task_output_distribution::Kind::RoundRobin( + r#gen::TaskOutputRoundRobinDistribution { channels }, )) => Ok(TaskOutputDistribution::RoundRobin { channels: channels as usize, }), - Some(gen::task_output_distribution::Kind::RoundRobinRow( - gen::TaskOutputRoundRobinRowDistribution { channels }, + Some(r#gen::task_output_distribution::Kind::RoundRobinRow( + r#gen::TaskOutputRoundRobinRowDistribution { channels }, )) => Ok(TaskOutputDistribution::RoundRobinRow { channels: channels as usize, }), @@ -464,35 +459,33 @@ impl TryFrom for TaskOutputDistribution { } } -impl From for gen::TaskOutputLocator { +impl From for r#gen::TaskOutputLocator { fn from(value: TaskOutputLocator) -> Self { let kind = match value { TaskOutputLocator::Local { replicas } => { - gen::task_output_locator::Kind::Local(gen::TaskOutputLocalLocator { + r#gen::task_output_locator::Kind::Local(r#gen::TaskOutputLocalLocator { replicas: replicas as u64, }) } - TaskOutputLocator::Remote { uri } => { - gen::task_output_locator::Kind::Remote(gen::TaskOutputRemoteLocator { uri }) + TaskOutputLocator::Remote => { + r#gen::task_output_locator::Kind::Remote(r#gen::TaskOutputRemoteLocator {}) } }; - gen::TaskOutputLocator { kind: Some(kind) } + r#gen::TaskOutputLocator { kind: Some(kind) } } } -impl TryFrom for TaskOutputLocator { +impl TryFrom for TaskOutputLocator { type Error = ExecutionError; - fn try_from(value: gen::TaskOutputLocator) -> Result { + fn try_from(value: r#gen::TaskOutputLocator) -> Result { match value.kind { - Some(gen::task_output_locator::Kind::Local(gen::TaskOutputLocalLocator { + Some(r#gen::task_output_locator::Kind::Local(r#gen::TaskOutputLocalLocator { replicas, })) => Ok(TaskOutputLocator::Local { replicas: replicas as usize, }), - Some(gen::task_output_locator::Kind::Remote(gen::TaskOutputRemoteLocator { uri })) => { - Ok(TaskOutputLocator::Remote { uri }) - } + Some(r#gen::task_output_locator::Kind::Remote(_)) => Ok(TaskOutputLocator::Remote), None => Err(ExecutionError::InvalidArgument( "cannot decode empty task output locator".to_string(), )), @@ -536,12 +529,11 @@ impl TaskInput { .collect() }) .collect(), - TaskInputLocator::Remote { uri, stage, keys } => keys + TaskInputLocator::Remote { stage, keys } => keys .iter() .map(|keys| { keys.iter() .map(|key| TaskReadLocation::Remote { - uri: uri.clone(), key: TaskStreamKey { job_id, stage: *stage, @@ -583,9 +575,8 @@ impl TaskOutput { }, }) .collect(), - TaskOutputLocator::Remote { uri } => (0..channels) + TaskOutputLocator::Remote => (0..channels) .map(|channel| TaskWriteLocation::Remote { - uri: uri.clone(), key: TaskStreamKey { job_id: key.job_id, stage: key.stage, @@ -609,7 +600,7 @@ impl TaskOutput { let keys = keys .iter() .map(|k| { - try_decode_physical_expr(ctx, codec, k.as_ref(), schema) + decode_remote_physical_expr(ctx, codec, k.as_ref(), schema) .map_err(|e| e.into()) }) .collect::>>()?; diff --git a/crates/sail-execution/src/task/mod.rs b/crates/sail-execution/src/task/mod.rs index 0aa984852..9b6d56216 100644 --- a/crates/sail-execution/src/task/mod.rs +++ b/crates/sail-execution/src/task/mod.rs @@ -1,6 +1,6 @@ pub mod definition; pub mod scheduling; -pub mod gen { +pub mod r#gen { tonic::include_proto!("sail.task"); } diff --git a/crates/sail-execution/src/task_runner/core.rs b/crates/sail-execution/src/task_runner/core.rs index 86813b3a8..d6be6439f 100644 --- a/crates/sail-execution/src/task_runner/core.rs +++ b/crates/sail-execution/src/task_runner/core.rs @@ -4,26 +4,25 @@ use std::sync::Arc; use datafusion::catalog::memory::DataSourceExec; use datafusion::common::internal_err; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; -use datafusion::datasource::physical_plan::{FileScanConfigBuilder, ParquetSource}; +use datafusion::datasource::physical_plan::{FileScanConfig, FileScanConfigBuilder, ParquetSource}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr_adapter::PhysicalExprAdapterFactory; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use log::debug; +use sail_common::actor::{Actor, ActorContext}; use sail_common_datafusion::error::CommonErrorCause; use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapterFactory; use sail_python_udf::error::PyErrExtractor; -use sail_server::actor::{Actor, ActorContext}; use sail_telemetry::telemetry::global_metrics; -use sail_telemetry::{trace_execution_plan, TracingExecOptions}; +use sail_telemetry::{TracingExecOptions, trace_execution_plan}; use tokio::sync::oneshot; use crate::driver::TaskStatus; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::{TaskKey, TaskKeyDisplay}; use crate::plan::{ShuffleReadExec, ShuffleWriteExec, StageInputExec}; -use crate::proto::codec::RemoteExecutionCodec; -use crate::proto::decode::try_decode_physical_plan; +use crate::proto::{RemoteExecutionCodec, decode_remote_physical_plan}; use crate::stream_accessor::{StreamAccessor, StreamAccessorMessage}; use crate::task::definition::{TaskDefinition, TaskInput, TaskOutput}; use crate::task_runner::monitor::TaskMonitor; @@ -84,15 +83,15 @@ impl TaskRunner { T::Message: TaskRunnerMessage + StreamAccessorMessage, { let plan = - try_decode_physical_plan(&context, self.codec.as_ref(), definition.plan.as_ref())?; - let plan = self.rewrite_parquet_adapters(plan)?; + decode_remote_physical_plan(&context, self.codec.as_ref(), definition.plan.as_ref())?; + let plan = self.rewrite_file_scans(plan)?; let plan = self.rewrite_shuffle( ctx, key, &definition.inputs, &definition.output, plan, - &context, + context.clone(), )?; debug!( "{} execution plan\n{}", @@ -111,21 +110,31 @@ impl TaskRunner { Ok(stream) } - fn rewrite_parquet_adapters( + fn rewrite_file_scans( &mut self, plan: Arc, ) -> ExecutionResult> { let result = plan.transform(|node| { - if let Some(ds) = node.downcast_ref::() { - if let Some((base_config, _parquet)) = ds.downcast_to_file_source::() + if let Some(ds) = node.downcast_ref::() + && let Some(base_config) = ds.data_source().downcast_ref::() + { + // DataFusion file scans can use process-local sibling state to let + // partitions steal work from a shared queue of all file groups. In Sail + // cluster mode each partition runs as an isolated task with its own + // deserialized plan, so that queue would be recreated in every task and + // every task would scan every file. Preserve-order disables sibling + // work sharing and keeps each task on its own file group. + let mut builder = + FileScanConfigBuilder::from(base_config.clone()).with_preserve_order(true); + if ds.downcast_to_file_source::().is_some() + && base_config.expr_adapter_factory.is_none() { let adapter_factory: Arc = Arc::new(SchemaEvolutionPhysicalExprAdapterFactory {}); - let builder = FileScanConfigBuilder::from(base_config.clone()) - .with_expr_adapter(Some(adapter_factory)); - let new_exec = DataSourceExec::from_data_source(builder.build()); - return Ok(Transformed::yes(new_exec as Arc)); + builder = builder.with_expr_adapter(Some(adapter_factory)); } + let new_exec = DataSourceExec::from_data_source(builder.build()); + return Ok(Transformed::yes(new_exec as Arc)); } Ok(Transformed::no(node)) }); @@ -139,36 +148,37 @@ impl TaskRunner { inputs: &[TaskInput], output: &TaskOutput, plan: Arc, - context: &TaskContext, + context: Arc, ) -> ExecutionResult> where T::Message: TaskRunnerMessage + StreamAccessorMessage, { - let handle = ctx.handle(); - let result = plan.transform(move |node| { - if let Some(placeholder) = node.downcast_ref::>() { - let Some(input) = inputs.get(*placeholder.input()) else { - return internal_err!( - "stage input index {} out of bounds for {}", - placeholder.input(), - TaskKeyDisplay(key) + let accessor = StreamAccessor::new(ctx.handle().clone(), context.clone()); + let result = { + let accessor = accessor.clone(); + plan.transform(move |node| { + if let Some(placeholder) = node.downcast_ref::>() { + let Some(input) = inputs.get(*placeholder.input()) else { + return internal_err!( + "stage input index {} out of bounds for {}", + placeholder.input(), + TaskKeyDisplay(key) + ); + }; + let locations = input.locations(key.job_id); + let shuffle = ShuffleReadExec::new( + locations, + Arc::new(accessor.clone()), + placeholder.properties().clone(), ); - }; - let locations = input.locations(key.job_id); - let accessor = StreamAccessor::new(handle.clone()); - let shuffle = ShuffleReadExec::new( - locations, - Arc::new(accessor), - placeholder.properties().clone(), - ); - Ok(Transformed::yes(Arc::new(shuffle))) - } else { - Ok(Transformed::no(node)) - } - }); + Ok(Transformed::yes(Arc::new(shuffle))) + } else { + Ok(Transformed::no(node)) + } + }) + }; let plan = result.data()?; let schema = plan.schema(); - let accessor = StreamAccessor::new(handle.clone()); let mut locations = vec![vec![]; plan.output_partitioning().partition_count()]; match locations.get_mut(key.partition) { Some(x) => x.extend(output.locations(key)), @@ -179,7 +189,7 @@ impl TaskRunner { ))); } }; - let partitioning = output.partitioning(context, &schema, self.codec.as_ref())?; + let partitioning = output.partitioning(&context, &schema, self.codec.as_ref())?; let row_based = output.row_based(); let shuffle = ShuffleWriteExec::new(plan, locations, Arc::new(accessor), partitioning, row_based); diff --git a/crates/sail-execution/src/task_runner/monitor.rs b/crates/sail-execution/src/task_runner/monitor.rs index 65fe5a4ef..259ff5cc1 100644 --- a/crates/sail-execution/src/task_runner/monitor.rs +++ b/crates/sail-execution/src/task_runner/monitor.rs @@ -1,8 +1,8 @@ use datafusion::execution::SendableRecordBatchStream; use futures::StreamExt; +use sail_common::actor::{Actor, ActorHandle}; use sail_common_datafusion::error::CommonErrorCause; use sail_python_udf::error::PyErrExtractor; -use sail_server::actor::{Actor, ActorHandle}; use tokio::sync::oneshot; use crate::driver::TaskStatus; @@ -71,7 +71,7 @@ where /// Drains the output stream and builds a succeeded or failed status message. async fn execute(key: TaskKey, mut stream: SendableRecordBatchStream) -> T::Message { - let event = loop { + loop { let Some(batch) = stream.next().await else { break T::Message::report_task_status( key.clone(), @@ -95,7 +95,6 @@ where Some(cause), ); } - }; - event + } } } diff --git a/crates/sail-execution/src/worker/actor/core.rs b/crates/sail-execution/src/worker/actor/core.rs index 7d8f3d2c8..65b2f9d5f 100644 --- a/crates/sail-execution/src/worker/actor/core.rs +++ b/crates/sail-execution/src/worker/actor/core.rs @@ -1,18 +1,18 @@ use std::mem; -use fastrace::future::FutureExt; use fastrace::Span; +use fastrace::future::FutureExt; use log::info; -use sail_server::actor::{Actor, ActorAction, ActorContext}; +use sail_common::actor::{Actor, ActorAction, ActorContext}; use crate::driver::DriverClientSet; use crate::rpc::{ClientOptions, ServerMonitor}; use crate::stream_manager::{StreamManager, StreamManagerOptions}; use crate::task_runner::TaskRunner; +use crate::worker::WorkerActor; use crate::worker::event::WorkerEvent; use crate::worker::options::WorkerOptions; use crate::worker::peer_tracker::{PeerTracker, PeerTrackerOptions}; -use crate::worker::WorkerActor; #[tonic::async_trait] impl Actor for WorkerActor { @@ -24,11 +24,14 @@ impl Actor for WorkerActor { } fn new(options: WorkerOptions) -> Self { - let driver_client_set = DriverClientSet::new(ClientOptions { - enable_tls: options.enable_tls, - host: options.driver_host.clone(), - port: options.driver_port, - }); + let driver_client_set = DriverClientSet::new( + options.driver_id, + ClientOptions { + enable_tls: options.enable_tls, + host: options.driver_host.clone(), + port: options.driver_port, + }, + ); let peer_tracker = PeerTracker::new(PeerTrackerOptions::from(&options)); let stream_manager = StreamManager::new(StreamManagerOptions::from(&options)); Self { @@ -85,11 +88,11 @@ impl Actor for WorkerActor { result, } => self.handle_create_local_stream(ctx, key, storage, schema, result), WorkerEvent::CreateRemoteStream { - uri, key, schema, + context, result, - } => self.handle_create_remote_stream(ctx, uri, key, schema, result), + } => self.handle_create_remote_stream(ctx, key, schema, context, result), WorkerEvent::FetchDriverStream { key, schema, @@ -99,11 +102,11 @@ impl Actor for WorkerActor { self.handle_fetch_worker_stream(ctx, owner, key, result) } WorkerEvent::FetchRemoteStream { - uri, key, schema, + context, result, - } => self.handle_fetch_remote_stream(ctx, uri, key, schema, result), + } => self.handle_fetch_remote_stream(ctx, key, schema, context, result), WorkerEvent::CleanUpJob { job_id, stage } => { self.handle_clean_up_job(ctx, job_id, stage) } diff --git a/crates/sail-execution/src/worker/actor/handler.rs b/crates/sail-execution/src/worker/actor/handler.rs index cabce0af2..c9d7eafc9 100644 --- a/crates/sail-execution/src/worker/actor/handler.rs +++ b/crates/sail-execution/src/worker/actor/handler.rs @@ -1,9 +1,11 @@ use std::mem; +use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; use log::{error, info, warn}; +use sail_common::actor::{ActorAction, ActorContext}; use sail_common_datafusion::error::CommonErrorCause; -use sail_server::actor::{ActorAction, ActorContext}; use tokio::sync::oneshot; use crate::driver::TaskStatus; @@ -12,9 +14,9 @@ use crate::id::{JobId, TaskKey, TaskStreamKey, WorkerId}; use crate::stream::reader::TaskStreamSource; use crate::stream::writer::{LocalStreamStorage, TaskStreamSink}; use crate::task::definition::TaskDefinition; +use crate::worker::WorkerEvent; use crate::worker::actor::WorkerActor; use crate::worker::event::{WorkerLocation, WorkerStreamOwner}; -use crate::worker::WorkerEvent; impl WorkerActor { pub(super) fn handle_server_ready( @@ -191,12 +193,15 @@ impl WorkerActor { pub(super) fn handle_create_remote_stream( &mut self, _ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, ) -> ActorAction { - let _ = result.send(self.stream_manager.create_remote_stream(uri, key, schema)); + let _ = result.send( + self.stream_manager + .create_remote_stream(key, schema, &context), + ); ActorAction::Continue } @@ -252,14 +257,14 @@ impl WorkerActor { pub(super) fn handle_fetch_remote_stream( &mut self, ctx: &mut ActorContext, - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, ) -> ActorAction { let _ = result.send( self.stream_manager - .fetch_remote_stream(ctx, uri, &key, schema), + .fetch_remote_stream(ctx, &key, schema, &context), ); ActorAction::Continue } diff --git a/crates/sail-execution/src/worker/actor/mod.rs b/crates/sail-execution/src/worker/actor/mod.rs index 586db5b44..1cef67a71 100644 --- a/crates/sail-execution/src/worker/actor/mod.rs +++ b/crates/sail-execution/src/worker/actor/mod.rs @@ -6,8 +6,8 @@ use crate::driver::DriverClientSet; use crate::rpc::ServerMonitor; use crate::stream_manager::StreamManager; use crate::task_runner::TaskRunner; -use crate::worker::peer_tracker::PeerTracker; use crate::worker::WorkerOptions; +use crate::worker::peer_tracker::PeerTracker; pub struct WorkerActor { options: WorkerOptions, diff --git a/crates/sail-execution/src/worker/actor/rpc.rs b/crates/sail-execution/src/worker/actor/rpc.rs index 0c2b49df5..466d4582a 100644 --- a/crates/sail-execution/src/worker/actor/rpc.rs +++ b/crates/sail-execution/src/worker/actor/rpc.rs @@ -1,7 +1,7 @@ use arrow_flight::flight_service_server::FlightServiceServer; +use sail_common::actor::ActorHandle; use sail_common::config::GRPC_MAX_MESSAGE_LENGTH_DEFAULT; -use sail_server::actor::ActorHandle; -use sail_server::ServerBuilder; +use sail_common::server::ServerBuilder; use tokio::net::{TcpListener, ToSocketAddrs}; use tokio::sync::oneshot::Sender; use tonic::async_trait; @@ -11,18 +11,18 @@ use crate::error::{ExecutionError, ExecutionResult}; use crate::id::TaskStreamKey; use crate::stream::reader::TaskStreamSource; use crate::stream_service::{TaskStreamFetcher, TaskStreamFlightServer}; +use crate::worker::WorkerEvent; use crate::worker::actor::WorkerActor; use crate::worker::event::WorkerStreamOwner; -use crate::worker::gen::worker_service_server::WorkerServiceServer; +use crate::worker::r#gen::worker_service_server::WorkerServiceServer; use crate::worker::server::WorkerServer; -use crate::worker::WorkerEvent; struct WorkerTaskStreamFetcher { handle: ActorHandle, } #[async_trait] -impl TaskStreamFetcher for WorkerTaskStreamFetcher { +impl TaskStreamFetcher for WorkerTaskStreamFetcher { async fn fetch( &self, key: TaskStreamKey, @@ -54,9 +54,10 @@ impl WorkerActor { .send_compressed(CompressionEncoding::Gzip) .send_compressed(CompressionEncoding::Zstd); - let flight_server = TaskStreamFlightServer::new(Box::new(WorkerTaskStreamFetcher { - handle: handle.clone(), - })); + let flight_server = + TaskStreamFlightServer::::new(Box::new(WorkerTaskStreamFetcher { + handle: handle.clone(), + })); let flight_service = FlightServiceServer::new(flight_server) .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT) .accept_compressed(CompressionEncoding::Gzip) @@ -69,7 +70,7 @@ impl WorkerActor { .await?; ServerBuilder::new("sail_worker", Default::default()) - .add_service(service, Some(crate::worker::gen::FILE_DESCRIPTOR_SET)) + .add_service(service, Some(crate::worker::r#gen::FILE_DESCRIPTOR_SET)) .await .add_service(flight_service, None) .await diff --git a/crates/sail-execution/src/worker/client.rs b/crates/sail-execution/src/worker/client.rs index 04cc5df0e..f2e2b6ac5 100644 --- a/crates/sail-execution/src/worker/client.rs +++ b/crates/sail-execution/src/worker/client.rs @@ -3,11 +3,11 @@ use prost::Message; use crate::error::ExecutionResult; use crate::id::{JobId, TaskKey}; use crate::rpc::{ClientHandle, ClientOptions, ClientService}; -use crate::stream_service::TaskStreamFlightClient; +use crate::stream_service::{TaskStreamFlightClient, TaskStreamOwner}; use crate::task::definition::TaskDefinition; use crate::worker::event::WorkerLocation; -use crate::worker::gen::worker_service_client::WorkerServiceClient; -use crate::worker::gen::{ +use crate::worker::r#gen::worker_service_client::WorkerServiceClient; +use crate::worker::r#gen::{ CleanUpJobRequest, CleanUpJobResponse, RunTaskRequest, RunTaskResponse, StopTaskRequest, StopTaskResponse, StopWorkerRequest, StopWorkerResponse, }; @@ -22,7 +22,7 @@ impl WorkerClientSet { pub fn new(options: ClientOptions) -> Self { Self { core: WorkerClient::new(options.clone()), - flight: TaskStreamFlightClient::new(options), + flight: TaskStreamFlightClient::new(options, TaskStreamOwner::Worker), } } } @@ -48,7 +48,7 @@ impl WorkerClient { definition: TaskDefinition, peers: Vec, ) -> ExecutionResult<()> { - let definition = crate::task::gen::TaskDefinition::from(definition).encode_to_vec(); + let definition = crate::task::r#gen::TaskDefinition::from(definition).encode_to_vec(); let request = RunTaskRequest { job_id: key.job_id.into(), stage: key.stage as u64, diff --git a/crates/sail-execution/src/worker/debug.rs b/crates/sail-execution/src/worker/debug.rs index fe79cadd9..5daf7b24d 100644 --- a/crates/sail-execution/src/worker/debug.rs +++ b/crates/sail-execution/src/worker/debug.rs @@ -1,8 +1,8 @@ use std::fmt::{Debug, Formatter}; -use sail_common::debug::DebugBinary; +use sail_common::utils::debug::DebugBinary; -use crate::worker::gen::RunTaskRequest; +use crate::worker::r#gen::RunTaskRequest; impl Debug for RunTaskRequest { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { diff --git a/crates/sail-execution/src/worker/entrypoint.rs b/crates/sail-execution/src/worker/entrypoint.rs index c99b30c3c..3cde64a08 100644 --- a/crates/sail-execution/src/worker/entrypoint.rs +++ b/crates/sail-execution/src/worker/entrypoint.rs @@ -1,10 +1,10 @@ use datafusion::prelude::SessionContext; -use fastrace::collector::SpanContext; use fastrace::Span; +use fastrace::collector::SpanContext; +use sail_common::actor::ActorSystem; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeHandle; -use sail_server::actor::ActorSystem; -use sail_telemetry::common::{ContextPropagationEnv, SpanAttribute}; +use sail_common::telemetry::{ContextPropagationEnv, SpanAttribute}; use crate::error::ExecutionError; use crate::worker::{WorkerActor, WorkerOptions}; diff --git a/crates/sail-execution/src/worker/event.rs b/crates/sail-execution/src/worker/event.rs index 2571aa515..befc79d4c 100644 --- a/crates/sail-execution/src/worker/event.rs +++ b/crates/sail-execution/src/worker/event.rs @@ -1,8 +1,10 @@ use std::borrow::Cow; +use std::sync::Arc; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::execution::TaskContext; +use sail_common::telemetry::{SpanAssociation, SpanAttribute}; use sail_common_datafusion::error::CommonErrorCause; -use sail_telemetry::common::{SpanAssociation, SpanAttribute}; use tokio::sync::oneshot; use crate::driver::TaskStatus; @@ -11,7 +13,7 @@ use crate::id::{JobId, TaskKey, TaskStreamKey, WorkerId}; use crate::stream::reader::TaskStreamSource; use crate::stream::writer::{LocalStreamStorage, TaskStreamSink}; use crate::task::definition::TaskDefinition; -use crate::worker::gen; +use crate::worker::r#gen; pub enum WorkerEvent { ServerReady { @@ -48,9 +50,9 @@ pub enum WorkerEvent { result: oneshot::Sender>>, }, CreateRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>>, }, FetchDriverStream { @@ -64,9 +66,9 @@ pub enum WorkerEvent { result: oneshot::Sender>, }, FetchRemoteStream { - uri: String, key: TaskStreamKey, schema: SchemaRef, + context: Arc, result: oneshot::Sender>, }, CleanUpJob { @@ -210,7 +212,6 @@ impl SpanAssociation for WorkerEvent { )); } WorkerEvent::CreateRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -220,6 +221,7 @@ impl SpanAssociation for WorkerEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -227,7 +229,6 @@ impl SpanAssociation for WorkerEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } WorkerEvent::FetchDriverStream { key: @@ -269,7 +270,6 @@ impl SpanAssociation for WorkerEvent { p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); } WorkerEvent::FetchRemoteStream { - uri, key: TaskStreamKey { job_id, @@ -279,6 +279,7 @@ impl SpanAssociation for WorkerEvent { channel, }, schema: _, + context: _, result: _, } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -286,7 +287,6 @@ impl SpanAssociation for WorkerEvent { p.push((SpanAttribute::EXECUTION_PARTITION, partition.to_string())); p.push((SpanAttribute::EXECUTION_ATTEMPT, attempt.to_string())); p.push((SpanAttribute::EXECUTION_CHANNEL, channel.to_string())); - p.push((SpanAttribute::EXECUTION_STREAM_REMOTE_URI, uri.clone())); } WorkerEvent::CleanUpJob { job_id, stage } => { p.push((SpanAttribute::EXECUTION_JOB_ID, job_id.to_string())); @@ -306,7 +306,7 @@ pub struct WorkerLocation { pub port: u16, } -impl From for gen::WorkerLocation { +impl From for r#gen::WorkerLocation { fn from(value: WorkerLocation) -> Self { Self { worker_id: value.worker_id.into(), @@ -316,10 +316,10 @@ impl From for gen::WorkerLocation { } } -impl TryFrom for WorkerLocation { +impl TryFrom for WorkerLocation { type Error = ExecutionError; - fn try_from(value: gen::WorkerLocation) -> Result { + fn try_from(value: r#gen::WorkerLocation) -> Result { let port = u16::try_from(value.port).map_err(|_| { ExecutionError::InvalidArgument(format!("invalid port: {}", value.port)) })?; diff --git a/crates/sail-execution/src/worker/mod.rs b/crates/sail-execution/src/worker/mod.rs index 31e6794bc..138a91ca4 100644 --- a/crates/sail-execution/src/worker/mod.rs +++ b/crates/sail-execution/src/worker/mod.rs @@ -8,7 +8,7 @@ mod peer_tracker; mod server; #[expect(clippy::allow_attributes)] -mod gen { +mod r#gen { tonic::include_proto!("sail.worker"); pub const FILE_DESCRIPTOR_SET: &[u8] = @@ -18,5 +18,5 @@ mod gen { pub(crate) use actor::WorkerActor; pub(crate) use client::WorkerClientSet; pub(crate) use event::{WorkerEvent, WorkerLocation, WorkerStreamOwner}; -pub(crate) use gen::worker_service_client::WorkerServiceClient; +pub(crate) use r#gen::worker_service_client::WorkerServiceClient; pub(crate) use options::WorkerOptions; diff --git a/crates/sail-execution/src/worker/options.rs b/crates/sail-execution/src/worker/options.rs index 535b40231..ee1f4f9ab 100644 --- a/crates/sail-execution/src/worker/options.rs +++ b/crates/sail-execution/src/worker/options.rs @@ -3,17 +3,20 @@ use std::time::Duration; use datafusion::prelude::SessionContext; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeHandle; -use sail_server::RetryStrategy; +use sail_common::utils::retry::RetryStrategy; -use crate::id::WorkerId; +use crate::id::{DriverId, WorkerId}; +use crate::shuffle::ShuffleBackendKind; use crate::worker_manager::WorkerLaunchOptions; #[readonly::make] pub struct WorkerOptions { - pub worker_id: WorkerId, pub enable_tls: bool, + pub session_id: String, + pub driver_id: DriverId, pub driver_host: String, pub driver_port: u16, + pub worker_id: WorkerId, pub worker_listen_host: String, pub worker_listen_port: u16, pub worker_external_host: String, @@ -22,6 +25,7 @@ pub struct WorkerOptions { pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackendKind, pub runtime: RuntimeHandle, pub session: SessionContext, } @@ -29,14 +33,16 @@ pub struct WorkerOptions { impl WorkerOptions { pub fn new(config: &AppConfig, runtime: RuntimeHandle, session: SessionContext) -> Self { Self { - worker_id: config.cluster.worker_id.into(), enable_tls: config.cluster.enable_tls, + session_id: config.cluster.session_id.clone(), + driver_id: config.cluster.driver_id.into(), driver_host: config.cluster.driver_external_host.clone(), driver_port: if config.cluster.driver_external_port > 0 { config.cluster.driver_external_port } else { config.cluster.driver_listen_port }, + worker_id: config.cluster.worker_id.into(), worker_listen_host: config.cluster.worker_listen_host.clone(), worker_listen_port: config.cluster.worker_listen_port, worker_external_host: config.cluster.worker_external_host.clone(), @@ -49,6 +55,7 @@ impl WorkerOptions { config.cluster.task_stream_creation_timeout_secs, ), rpc_retry_strategy: (&config.cluster.rpc_retry_strategy).into(), + shuffle_backend: (&config.cluster.shuffle_backend).into(), runtime, session, } @@ -61,10 +68,12 @@ impl WorkerOptions { session: SessionContext, ) -> Self { WorkerOptions { - worker_id: id, enable_tls: options.enable_tls, + session_id: options.session_id, + driver_id: options.driver_id, driver_host: options.driver_external_host, driver_port: options.driver_external_port, + worker_id: id, worker_listen_host: "127.0.0.1".to_string(), worker_listen_port: 0, worker_external_host: "127.0.0.1".to_string(), @@ -73,6 +82,7 @@ impl WorkerOptions { task_stream_buffer: options.task_stream_buffer, task_stream_creation_timeout: options.task_stream_creation_timeout, rpc_retry_strategy: options.rpc_retry_strategy, + shuffle_backend: options.shuffle_backend, runtime: runtime.clone(), session, } diff --git a/crates/sail-execution/src/worker/peer_tracker/core.rs b/crates/sail-execution/src/worker/peer_tracker/core.rs index 68a2286ce..d094f01e3 100644 --- a/crates/sail-execution/src/worker/peer_tracker/core.rs +++ b/crates/sail-execution/src/worker/peer_tracker/core.rs @@ -1,5 +1,5 @@ use datafusion::common::HashMap; -use sail_server::actor::ActorContext; +use sail_common::actor::ActorContext; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::WorkerId; diff --git a/crates/sail-execution/src/worker/server.rs b/crates/sail-execution/src/worker/server.rs index d1d1c81ba..4d75ad26a 100644 --- a/crates/sail-execution/src/worker/server.rs +++ b/crates/sail-execution/src/worker/server.rs @@ -1,18 +1,18 @@ use log::debug; use prost::Message; -use sail_server::actor::ActorHandle; +use sail_common::actor::ActorHandle; use tonic::{Request, Response, Status}; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::TaskKey; use crate::task::definition::TaskDefinition; +use crate::worker::WorkerEvent; use crate::worker::actor::WorkerActor; -use crate::worker::gen::worker_service_server::WorkerService; -use crate::worker::gen::{ +use crate::worker::r#gen::worker_service_server::WorkerService; +use crate::worker::r#gen::{ CleanUpJobRequest, CleanUpJobResponse, RunTaskRequest, RunTaskResponse, StopTaskRequest, StopTaskResponse, StopWorkerRequest, StopWorkerResponse, }; -use crate::worker::WorkerEvent; pub struct WorkerServer { handle: ActorHandle, @@ -44,7 +44,7 @@ impl WorkerService for WorkerServer { .into_iter() .map(|x| x.try_into()) .collect::>>()?; - let definition = crate::task::gen::TaskDefinition::decode(definition.as_slice()) + let definition = crate::task::r#gen::TaskDefinition::decode(definition.as_slice()) .map_err(|e| Status::invalid_argument(format!("invalid task definition: {e}")))?; let event = WorkerEvent::RunTask { key: TaskKey { diff --git a/crates/sail-execution/src/worker_manager/kubernetes.rs b/crates/sail-execution/src/worker_manager/kubernetes.rs index 53e5031da..738dd4bdf 100644 --- a/crates/sail-execution/src/worker_manager/kubernetes.rs +++ b/crates/sail-execution/src/worker_manager/kubernetes.rs @@ -1,23 +1,27 @@ use std::collections::BTreeMap; use std::env; +use std::sync::Arc; use fastrace::collector::SpanContext; +use futures::future::BoxFuture; use k8s_openapi::api::core::v1::{ Container, EnvVar, EnvVarSource, ObjectFieldSelector, Pod, PodSpec, PodTemplateSpec, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; use k8s_openapi::{DeepMerge, Resource}; -use kube::api::{DeleteParams, ListParams}; use kube::Api; -use rand::distr::Uniform; +use kube::api::{DeleteParams, ListParams}; use rand::RngExt; +use rand::distr::Uniform; +use sail_common::actor::ActorSystem; use sail_common::config::ClusterConfigEnv; -use sail_server::RetryStrategy; -use sail_telemetry::common::ContextPropagationEnv; +use sail_common::telemetry::ContextPropagationEnv; +use sail_common::utils::retry::RetryStrategy; use tokio::sync::OnceCell; use crate::error::{ExecutionError, ExecutionResult}; use crate::id::WorkerId; +use crate::shuffle::{ShuffleBackendKind, ShuffleCompression}; use crate::worker_manager::{WorkerLaunchOptions, WorkerManager}; #[derive(Debug, Clone)] @@ -31,15 +35,27 @@ pub struct KubernetesWorkerManagerOptions { pub worker_pod_template: String, } -pub struct KubernetesWorkerManager { +pub struct KubernetesWorkerService { /// An opaque name that can be used to create names to uniquely identify Kubernetes resources. name: String, options: KubernetesWorkerManagerOptions, pods: OnceCell>, } +pub struct KubernetesWorkerManager { + service: Arc, +} + impl KubernetesWorkerManager { pub fn new(options: KubernetesWorkerManagerOptions) -> Self { + Self { + service: Arc::new(KubernetesWorkerService::new(options)), + } + } +} + +impl KubernetesWorkerService { + fn new(options: KubernetesWorkerManagerOptions) -> Self { Self { name: Self::generate_name(), options, @@ -47,7 +63,7 @@ impl KubernetesWorkerManager { } } - pub fn generate_name() -> String { + fn generate_name() -> String { #[expect(clippy::unwrap_used)] rand::rng() .sample_iter(Uniform::new(0, 36).unwrap()) @@ -101,7 +117,7 @@ impl KubernetesWorkerManager { ), ( "app.kubernetes.io/instance".to_string(), - format!("{}-{}", self.name, id), + format!("{}-{id}", self.name), ), ( "sail.lakesail.com/worker-manager".to_string(), @@ -113,12 +129,15 @@ impl KubernetesWorkerManager { fn build_pod_env(&self, id: WorkerId, options: WorkerLaunchOptions) -> Vec { let WorkerLaunchOptions { enable_tls, + driver_id, + session_id, driver_external_host, driver_external_port, worker_heartbeat_interval, task_stream_buffer, task_stream_creation_timeout, rpc_retry_strategy, + shuffle_backend, } = options; let w3c_traceparent = SpanContext::current_local_parent().map(|x| x.encode_w3c_traceparent()); @@ -169,6 +188,16 @@ impl KubernetesWorkerManager { value: Some(driver_external_port.to_string()), value_from: None, }, + EnvVar { + name: ClusterConfigEnv::DRIVER_ID.to_string(), + value: Some(u64::from(driver_id).to_string()), + value_from: None, + }, + EnvVar { + name: ClusterConfigEnv::SESSION_ID.to_string(), + value: Some(session_id), + value_from: None, + }, EnvVar { name: ClusterConfigEnv::WORKER_ID.to_string(), value: Some(u64::from(id).to_string()), @@ -210,7 +239,51 @@ impl KubernetesWorkerManager { value: Some(rpc_retry_strategy), value_from: None, }, + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__TYPE.to_string(), + value: Some( + match &shuffle_backend { + ShuffleBackendKind::Flight => "flight", + ShuffleBackendKind::Storage { .. } => "storage", + } + .to_string(), + ), + value_from: None, + }, ]; + if let ShuffleBackendKind::Storage { + path, + max_file_size, + compression, + } = shuffle_backend + { + if let Some(path) = path { + env.push(EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__STORAGE__PATH.to_string(), + value: Some(path), + value_from: None, + }); + } + env.extend([ + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__STORAGE__MAX_FILE_SIZE.to_string(), + value: Some(max_file_size.to_string()), + value_from: None, + }, + EnvVar { + name: ClusterConfigEnv::SHUFFLE_BACKEND__STORAGE__COMPRESSION.to_string(), + value: Some( + match compression { + ShuffleCompression::None => "none", + ShuffleCompression::Lz4 => "lz4", + ShuffleCompression::Zstd => "zstd", + } + .to_string(), + ), + value_from: None, + }, + ]); + } if let Some(traceparent) = w3c_traceparent { env.push(EnvVar { name: ContextPropagationEnv::TRACEPARENT.to_string(), @@ -224,6 +297,22 @@ impl KubernetesWorkerManager { #[tonic::async_trait] impl WorkerManager for KubernetesWorkerManager { + fn launch_worker( + &self, + _system: &mut ActorSystem, + id: WorkerId, + options: WorkerLaunchOptions, + ) -> BoxFuture<'static, ExecutionResult<()>> { + let service = self.service.clone(); + Box::pin(async move { service.launch_worker(id, options).await }) + } + + async fn stop(&self) -> ExecutionResult<()> { + self.service.stop().await + } +} + +impl KubernetesWorkerService { async fn launch_worker( &self, id: WorkerId, @@ -255,10 +344,10 @@ impl WorkerManager for KubernetesWorkerManager { "failed to parse worker pod template: {e}", )) })?; - if let Some(metadata) = &template.metadata { - if let Some(template_labels) = &metadata.labels { - labels.extend(template_labels.clone()); - } + if let Some(metadata) = &template.metadata + && let Some(template_labels) = &metadata.labels + { + labels.extend(template_labels.clone()); } if let Some(s) = template.spec { spec.merge_from(s); @@ -324,10 +413,10 @@ mod tests { let mut labels = BTreeMap::new(); let parsed_template: PodTemplateSpec = serde_json::from_str(&template_json).unwrap(); - if let Some(metadata) = &parsed_template.metadata { - if let Some(template_labels) = &metadata.labels { - labels.extend(template_labels.clone()); - } + if let Some(metadata) = &parsed_template.metadata + && let Some(template_labels) = &metadata.labels + { + labels.extend(template_labels.clone()); } // Add default labels (simulating build_pod_labels) diff --git a/crates/sail-execution/src/worker_manager/local.rs b/crates/sail-execution/src/worker_manager/local.rs index 54d381fc4..ec4698e97 100644 --- a/crates/sail-execution/src/worker_manager/local.rs +++ b/crates/sail-execution/src/worker_manager/local.rs @@ -1,62 +1,38 @@ -use std::collections::HashMap; - use datafusion::prelude::SessionContext; +use futures::future::BoxFuture; +use sail_common::actor::ActorSystem; use sail_common::runtime::RuntimeHandle; -use sail_server::actor::{ActorHandle, ActorSystem}; -use tokio::sync::Mutex; use crate::error::ExecutionResult; use crate::id::WorkerId; use crate::worker::{WorkerActor, WorkerOptions}; use crate::worker_manager::{WorkerLaunchOptions, WorkerManager}; -struct LocalWorkerManagerState { - system: ActorSystem, - workers: HashMap>, -} - -impl LocalWorkerManagerState { - fn new() -> Self { - Self { - system: ActorSystem::new(), - workers: HashMap::new(), - } - } -} - pub struct LocalWorkerManager { - state: Mutex, runtime: RuntimeHandle, session: SessionContext, } impl LocalWorkerManager { pub fn new(runtime: RuntimeHandle, session: SessionContext) -> Self { - Self { - state: Mutex::new(LocalWorkerManagerState::new()), - runtime, - session, - } + Self { runtime, session } } } #[tonic::async_trait] impl WorkerManager for LocalWorkerManager { - async fn launch_worker( + fn launch_worker( &self, + system: &mut ActorSystem, id: WorkerId, options: WorkerLaunchOptions, - ) -> ExecutionResult<()> { + ) -> BoxFuture<'static, ExecutionResult<()>> { let options = WorkerOptions::local(id, options, self.runtime.clone(), self.session.clone()); - let mut state = self.state.lock().await; - let handle = state.system.spawn(options); - state.workers.insert(id, handle); - Ok(()) + system.spawn::(options); + Box::pin(std::future::ready(Ok(()))) } async fn stop(&self) -> ExecutionResult<()> { - let mut state = self.state.lock().await; - state.system.join().await; Ok(()) } } diff --git a/crates/sail-execution/src/worker_manager/mod.rs b/crates/sail-execution/src/worker_manager/mod.rs index c345aa5c5..044c53edc 100644 --- a/crates/sail-execution/src/worker_manager/mod.rs +++ b/crates/sail-execution/src/worker_manager/mod.rs @@ -2,19 +2,25 @@ mod kubernetes; mod local; mod options; +use futures::future::BoxFuture; pub(crate) use options::WorkerLaunchOptions; +use sail_common::actor::ActorSystem; use crate::error::ExecutionResult; use crate::id::WorkerId; #[tonic::async_trait] pub trait WorkerManager: Send + Sync + 'static { - /// Launch a worker. - async fn launch_worker( + /// Start launching a worker. + /// + /// Local workers are spawned into `system` immediately. The returned task launches an + /// external worker when necessary. + fn launch_worker( &self, + system: &mut ActorSystem, id: WorkerId, options: WorkerLaunchOptions, - ) -> ExecutionResult<()>; + ) -> BoxFuture<'static, ExecutionResult<()>>; /// Stop all workers on a best-effort basis. /// The driver have attempted to sent shutdown events to all workers diff --git a/crates/sail-execution/src/worker_manager/options.rs b/crates/sail-execution/src/worker_manager/options.rs index 642231072..95c7bce94 100644 --- a/crates/sail-execution/src/worker_manager/options.rs +++ b/crates/sail-execution/src/worker_manager/options.rs @@ -1,14 +1,20 @@ use std::time::Duration; -use sail_server::RetryStrategy; +use sail_common::utils::retry::RetryStrategy; + +use crate::id::DriverId; +use crate::shuffle::ShuffleBackendKind; #[derive(Debug, Clone)] pub struct WorkerLaunchOptions { pub enable_tls: bool, + pub session_id: String, + pub driver_id: DriverId, pub driver_external_host: String, pub driver_external_port: u16, pub worker_heartbeat_interval: Duration, pub task_stream_buffer: usize, pub task_stream_creation_timeout: Duration, pub rpc_retry_strategy: RetryStrategy, + pub shuffle_backend: ShuffleBackendKind, } diff --git a/crates/sail-flight/Cargo.toml b/crates/sail-flight/Cargo.toml index 9f2ee37c7..800ad0d7e 100644 --- a/crates/sail-flight/Cargo.toml +++ b/crates/sail-flight/Cargo.toml @@ -9,10 +9,10 @@ workspace = true [dependencies] sail-common = { path = "../sail-common" } sail-common-datafusion = { path = "../sail-common-datafusion" } +sail-execution = { path = "../sail-execution" } sail-catalog = { path = "../sail-catalog" } sail-catalog-memory = { path = "../sail-catalog-memory" } sail-plan = { path = "../sail-plan" } -sail-server = { path = "../sail-server" } sail-sql-analyzer = { path = "../sail-sql-analyzer" } sail-sql-parser = { path = "../sail-sql-parser" } sail-session = { path = "../sail-session" } diff --git a/crates/sail-flight/src/entrypoint.rs b/crates/sail-flight/src/entrypoint.rs new file mode 100644 index 000000000..f69c627b0 --- /dev/null +++ b/crates/sail-flight/src/entrypoint.rs @@ -0,0 +1,41 @@ +use std::future::Future; +use std::sync::Arc; + +use arrow_flight::flight_service_server::FlightServiceServer; +use sail_common::actor::ActorSystem; +use sail_common::config::AppConfig; +use sail_common::runtime::RuntimeHandle; +use sail_common::server::{ServerBuilder, ServerBuilderOptions}; +use tokio::net::TcpListener; + +use crate::service::SailFlightSqlService; +use crate::session::create_flight_session_manager; + +pub async fn serve( + listener: TcpListener, + signal: F, + config: Arc, + runtime: RuntimeHandle, +) -> Result<(), Box> +where + F: Future, +{ + let mut system = ActorSystem::new(); + let session_manager = create_flight_session_manager(config, runtime, &mut system).await?; + let result = { + let service = SailFlightSqlService::new(session_manager.clone()); + let flight_service = FlightServiceServer::new(service); + + let builder = ServerBuilder::new("flight_sql", ServerBuilderOptions::default()) + .add_service(flight_service, None) + .await; + + builder + .serve(listener, signal) + .await + .map_err(|e| std::io::Error::other(e.to_string())) + }; + session_manager.shutdown().await?; + system.join().await; + result.map_err(Into::into) +} diff --git a/crates/sail-flight/src/lib.rs b/crates/sail-flight/src/lib.rs index ff1a90176..55243bce0 100644 --- a/crates/sail-flight/src/lib.rs +++ b/crates/sail-flight/src/lib.rs @@ -1,37 +1,6 @@ +pub mod entrypoint; pub mod error; mod metrics; pub mod service; pub mod session; mod state; - -use std::future::Future; -use std::sync::Arc; - -use arrow_flight::flight_service_server::FlightServiceServer; -use sail_common::config::AppConfig; -use sail_common::runtime::RuntimeHandle; -use sail_server::{ServerBuilder, ServerBuilderOptions}; -use service::SailFlightSqlService; -use session::create_flight_session_manager; -use tokio::net::TcpListener; - -pub async fn serve( - listener: TcpListener, - signal: F, - config: Arc, - runtime: RuntimeHandle, -) -> Result<(), Box> -where - F: Future, -{ - let session_manager = create_flight_session_manager(config, runtime)?; - let service = SailFlightSqlService::new(session_manager); - let flight_service = FlightServiceServer::new(service); - - let builder = ServerBuilder::new("flight_sql", ServerBuilderOptions::default()) - .add_service(flight_service, None) - .await; - - builder.serve(listener, signal).await?; - Ok(()) -} diff --git a/crates/sail-flight/src/service.rs b/crates/sail-flight/src/service.rs index d2d6b4430..dc806189a 100644 --- a/crates/sail-flight/src/service.rs +++ b/crates/sail-flight/src/service.rs @@ -11,7 +11,7 @@ use arrow_flight::{ use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::prelude::SessionContext; -use futures::{stream, Stream, StreamExt}; +use futures::{Stream, StreamExt, stream}; use log::{debug, info}; use prost::Message; use sail_common::spec; @@ -92,7 +92,7 @@ impl FlightSqlService for SailFlightSqlService { query: CommandStatementQuery, request: Request, ) -> Result, Status> { - debug!("get_flight_info_statement: {}", &query.query); + debug!("get_flight_info_statement: {}", query.query); let statement = parse_one_statement(&query.query) .map_err(|e| Status::invalid_argument(format!("parse error: {e}")))?; diff --git a/crates/sail-flight/src/session.rs b/crates/sail-flight/src/session.rs index 4076aa585..53ef54a65 100644 --- a/crates/sail-flight/src/session.rs +++ b/crates/sail-flight/src/session.rs @@ -1,20 +1,21 @@ -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +use std::time::Duration; -use datafusion::common::{internal_datafusion_err, Result}; -use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::common::{Result, internal_datafusion_err}; use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::prelude::SessionConfig; +use sail_common::actor::ActorSystem; use sail_common::config::AppConfig; use sail_common::runtime::RuntimeHandle; use sail_common_datafusion::catalog::display::DefaultCatalogDisplay; use sail_common_datafusion::session::plan::PlanService; use sail_plan::catalog::SparkCatalogObjectDisplay; use sail_plan::formatter::SparkPlanFormatter; -use sail_server::actor::ActorSystem; use sail_session::session_factory::{ ServerSessionFactory, ServerSessionInfo, ServerSessionMutator, SessionFactory, }; -use sail_session::session_manager::{SessionManager, SessionManagerOptions}; +use sail_session::session_manager::{SessionManager, create_session_manager}; use crate::error::FlightError; @@ -56,32 +57,27 @@ impl ServerSessionMutator for FlightSessionMutator { fn create_flight_session_factory( config: Arc, runtime: RuntimeHandle, - system: Arc>, ) -> Box> { let mutator = Box::new(FlightSessionMutator { config: config.clone(), }); - Box::new(ServerSessionFactory::new(config, runtime, system, mutator)) + Box::new(ServerSessionFactory::new(config, runtime, mutator)) } -pub fn create_flight_session_manager( +pub async fn create_flight_session_manager( config: Arc, runtime: RuntimeHandle, + system: &mut ActorSystem, ) -> Result { - let system = Arc::new(Mutex::new(ActorSystem::new())); - let factory = { - let config = config.clone(); - let runtime = runtime.clone(); - let system = system.clone(); - Box::new(move || { - create_flight_session_factory(config.clone(), runtime.clone(), system.clone()) - }) - }; - let options = SessionManagerOptions::new(runtime.clone(), system, factory) - .with_session_timeout(std::time::Duration::from_secs( - config.flight.session_timeout_secs, - )); - SessionManager::try_new(options).map_err(|e| { + create_session_manager( + config.clone(), + runtime, + create_flight_session_factory, + Duration::from_secs(config.flight.session_timeout_secs), + system, + ) + .await + .map_err(|e| { FlightError::DataFusion(internal_datafusion_err!( "Failed to create session manager: {}", e diff --git a/crates/sail-function/src/aggregate/bitmap_construct_agg.rs b/crates/sail-function/src/aggregate/bitmap_construct_agg.rs index 35ddd3ce9..6cc3d4b21 100644 --- a/crates/sail-function/src/aggregate/bitmap_construct_agg.rs +++ b/crates/sail-function/src/aggregate/bitmap_construct_agg.rs @@ -8,8 +8,8 @@ use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; use datafusion::scalar::ScalarValue; use datafusion_common::types::{ - logical_int16, logical_int32, logical_int64, logical_int8, logical_uint16, logical_uint32, - logical_uint64, logical_uint8, NativeType, + NativeType, logical_int8, logical_int16, logical_int32, logical_int64, logical_uint8, + logical_uint16, logical_uint32, logical_uint64, }; use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; diff --git a/crates/sail-function/src/aggregate/count_min_sketch.rs b/crates/sail-function/src/aggregate/count_min_sketch.rs index 68de458d9..61d41892f 100644 --- a/crates/sail-function/src/aggregate/count_min_sketch.rs +++ b/crates/sail-function/src/aggregate/count_min_sketch.rs @@ -1,18 +1,16 @@ use std::fmt::Debug; -use arrow::array::{ - Array, ArrayRef, BinaryArray, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray, - StringArray, StringViewArray, -}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::array::{Array, ArrayRef, AsArray, BinaryArray}; +use arrow::datatypes::{DataType, Field, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type}; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::utils::format_state_name; use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, Signature, TypeSignature, Volatility, }; -use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err}; use super::utils::get_scalar_value; +use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err}; use crate::scalar::hash::utils::spark_compatible_murmur3_hash; use crate::scalar::math::java_random::JavaRandom; @@ -62,12 +60,14 @@ impl AggregateUDFImpl for CountMinSketchFunction { } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - Ok(vec![Field::new( - format_state_name(args.name, "count_min_sketch"), - DataType::Binary, - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, "count_min_sketch"), + DataType::Binary, + true, + ) + .into(), + ]) } } @@ -167,101 +167,59 @@ impl SparkCountMinSketch { } fn update_from_array(&mut self, values: &ArrayRef) -> Result<()> { - for row in 0..values.len() { - if values.is_null(row) { - continue; - } - match values.data_type() { - DataType::Int8 => { - let values = values.as_any().downcast_ref::().ok_or_else(|| { - DataFusionError::Internal("count_min_sketch expected Int8Array".to_string()) - })?; - self.add_long(values.value(row) as i64); - } - DataType::Int16 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected Int16Array".to_string(), - ) - })?; - self.add_long(values.value(row) as i64); + // Downcast once outside the row loop: the data type and the concrete array + // are loop-invariant, so dispatching per row would re-pay the match + downcast + // on every element of the batch. `iter().flatten()` skips nulls without a + // per-row branch. + match values.data_type() { + DataType::Int8 => { + for v in values.as_primitive::().iter().flatten() { + self.add_long(v as i64); } - DataType::Int32 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected Int32Array".to_string(), - ) - })?; - self.add_long(values.value(row) as i64); + } + DataType::Int16 => { + for v in values.as_primitive::().iter().flatten() { + self.add_long(v as i64); } - DataType::Int64 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected Int64Array".to_string(), - ) - })?; - self.add_long(values.value(row)); + } + DataType::Int32 => { + for v in values.as_primitive::().iter().flatten() { + self.add_long(v as i64); } - DataType::Binary => { - let values = - values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected BinaryArray".to_string(), - ) - })?; - self.add_binary(values.value(row)); + } + DataType::Int64 => { + for v in values.as_primitive::().iter().flatten() { + self.add_long(v); } - DataType::Utf8 => { - let values = - values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected StringArray".to_string(), - ) - })?; - self.add_binary(values.value(row).as_bytes()); + } + DataType::Binary => { + for v in values.as_binary::().iter().flatten() { + self.add_binary(v); } - DataType::LargeUtf8 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected LargeStringArray".to_string(), - ) - })?; - self.add_binary(values.value(row).as_bytes()); + } + DataType::Utf8 => { + for v in values.as_string::().iter().flatten() { + self.add_binary(v.as_bytes()); } - DataType::Utf8View => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "count_min_sketch expected StringViewArray".to_string(), - ) - })?; - self.add_binary(values.value(row).as_bytes()); + } + DataType::LargeUtf8 => { + for v in values.as_string::().iter().flatten() { + self.add_binary(v.as_bytes()); } - DataType::Null => {} - data_type => { - return exec_err!("count_min_sketch does not support input type {data_type}"); + } + DataType::Utf8View => { + for v in values.as_string_view().iter().flatten() { + self.add_binary(v.as_bytes()); } } + DataType::Null => {} + data_type => { + return Err(unsupported_data_type_exec_err( + "count_min_sketch", + "TINYINT, SMALLINT, INT, BIGINT, STRING or BINARY", + data_type, + )); + } } Ok(()) } @@ -275,8 +233,13 @@ impl SparkCountMinSketch { } fn add_binary(&mut self, item: &[u8]) { - let buckets = self.hash_binary(item); - for (row, bucket) in buckets.into_iter().enumerate() { + // Compute the two base hashes once, then derive each row's bucket inline + // (mirrors `add_long`) so a per-element bucket Vec is never allocated. + let hash1 = spark_compatible_murmur3_hash(item, 0) as i32; + let hash2 = spark_compatible_murmur3_hash(item, hash1 as u32) as i32; + for row in 0..self.depth { + let hash = hash1.wrapping_add((row as i32).wrapping_mul(hash2)); + let bucket = (hash % self.width as i32).wrapping_abs() as usize; self.table[row * self.width + bucket] += 1; } self.total_count += 1; @@ -357,17 +320,6 @@ impl SparkCountMinSketch { hash &= PRIME_MODULUS; (hash as i32 % self.width as i32) as usize } - - fn hash_binary(&self, item: &[u8]) -> Vec { - let hash1 = spark_compatible_murmur3_hash(item, 0) as i32; - let hash2 = spark_compatible_murmur3_hash(item, hash1 as u32) as i32; - (0..self.depth) - .map(|row| { - let hash = hash1.wrapping_add((row as i32).wrapping_mul(hash2)); - (hash % self.width as i32).wrapping_abs() as usize - }) - .collect() - } } fn validate_parameters(eps: f64, confidence: f64) -> Result<()> { @@ -384,10 +336,11 @@ fn validate_parameters(eps: f64, confidence: f64) -> Result<()> { fn validate_count_min_sketch_types(arg_types: &[DataType]) -> Result<()> { if arg_types.len() != 4 { - return exec_err!( - "count_min_sketch requires 4 arguments, got {}", - arg_types.len() - ); + return Err(invalid_arg_count_exec_err( + "count_min_sketch", + (4, 4), + arg_types.len(), + )); } match &arg_types[0] { DataType::Int8 @@ -400,18 +353,22 @@ fn validate_count_min_sketch_types(arg_types: &[DataType]) -> Result<()> { | DataType::Utf8View | DataType::Null => {} data_type => { - return exec_err!("count_min_sketch does not support input type {data_type}"); + return Err(unsupported_data_type_exec_err( + "count_min_sketch", + "TINYINT, SMALLINT, INT, BIGINT, STRING or BINARY", + data_type, + )); } } - if !matches!(arg_types[1], DataType::Float32 | DataType::Float64) { + if !matches!(arg_types[1], DataType::Float64) { return exec_err!( - "count_min_sketch requires a floating point eps argument, got {}", + "count_min_sketch requires a double eps argument, got {}", arg_types[1] ); } - if !matches!(arg_types[2], DataType::Float32 | DataType::Float64) { + if !matches!(arg_types[2], DataType::Float64) { return exec_err!( - "count_min_sketch requires a floating point confidence argument, got {}", + "count_min_sketch requires a double confidence argument, got {}", arg_types[2] ); } @@ -430,14 +387,15 @@ fn resolve_float64_literal(args: &AccumulatorArgs, index: usize, name: &str) -> })?; let value = get_scalar_value(value).map_err(|_| { DataFusionError::Plan(format!( - "count_min_sketch requires {name} to be a constant floating point value" + "count_min_sketch requires {name} to be a constant double value" )) })?; + // `validate_count_min_sketch_types` already rejects anything but Float64 for + // eps/confidence, so only the Float64 literal can reach here. match value { - ScalarValue::Float32(Some(value)) => Ok(value as f64), ScalarValue::Float64(Some(value)) => Ok(value), value => Err(DataFusionError::Plan(format!( - "count_min_sketch requires {name} to be a non-null floating point literal, got {}", + "count_min_sketch requires {name} to be a non-null double literal, got {}", value.data_type() ))), } diff --git a/crates/sail-function/src/aggregate/grouping_id.rs b/crates/sail-function/src/aggregate/grouping_id.rs index 126504674..5ec7b16a6 100644 --- a/crates/sail-function/src/aggregate/grouping_id.rs +++ b/crates/sail-function/src/aggregate/grouping_id.rs @@ -1,5 +1,5 @@ use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion::common::{not_impl_err, Result}; +use datafusion::common::{Result, not_impl_err}; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; @@ -37,7 +37,7 @@ impl AggregateUDFImpl for GroupingIdFunction { fn state_fields(&self, _args: StateFieldsArgs) -> Result> { Ok(vec![ - Field::new("grouping_id", DataType::Int64, false).into() + Field::new("grouping_id", DataType::Int64, false).into(), ]) } diff --git a/crates/sail-function/src/aggregate/histogram_numeric.rs b/crates/sail-function/src/aggregate/histogram_numeric.rs index bb0692321..02b7f111a 100644 --- a/crates/sail-function/src/aggregate/histogram_numeric.rs +++ b/crates/sail-function/src/aggregate/histogram_numeric.rs @@ -164,13 +164,13 @@ impl HistogramNumericFunction { | ScalarValue::UInt64(None) => { return Err(DataFusionError::Plan( "histogram_numeric requires a non-null integer literal for nbins".to_string(), - )) + )); } other => { return Err(DataFusionError::Plan(format!( "histogram_numeric requires an integer literal for nbins, got {}", other.data_type() - ))) + ))); } }; if nbins_i64 < 1 { diff --git a/crates/sail-function/src/aggregate/hll_sketch.rs b/crates/sail-function/src/aggregate/hll_sketch.rs index 23fe74c2f..753c827ac 100644 --- a/crates/sail-function/src/aggregate/hll_sketch.rs +++ b/crates/sail-function/src/aggregate/hll_sketch.rs @@ -7,12 +7,12 @@ use datafusion::logical_expr::utils::format_state_name; use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, Signature, TypeSignature, Volatility, }; -use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err}; use super::utils::get_scalar_value; use crate::hll_sketch::{ - empty_hll_union_bytes, new_hll_sketch, normalize_hll_sketch_bytes, union_hll_sketches, - update_hll_sketch_from_array, validate_lg_config_k, DEFAULT_LG_CONFIG_K, + DEFAULT_LG_CONFIG_K, empty_hll_union_bytes, new_hll_sketch, normalize_hll_sketch_bytes, + union_hll_sketches, update_hll_sketch_from_array, validate_lg_config_k, }; #[derive(Debug, PartialEq, Eq, Hash)] @@ -57,12 +57,14 @@ impl AggregateUDFImpl for HllSketchAggFunction { } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - Ok(vec![Field::new( - format_state_name(args.name, "hll_sketch"), - DataType::Binary, - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, "hll_sketch"), + DataType::Binary, + true, + ) + .into(), + ]) } } @@ -111,12 +113,14 @@ impl AggregateUDFImpl for HllUnionAggFunction { } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - Ok(vec![Field::new( - format_state_name(args.name, "hll_union"), - DataType::Binary, - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, "hll_union"), + DataType::Binary, + true, + ) + .into(), + ]) } } @@ -291,7 +295,7 @@ fn resolve_lg_config_k(args: &AccumulatorArgs, index: usize, function_name: &str return exec_err!( "{function_name} requires lgConfigK to be a non-null integer literal, got {}", value.data_type() - ) + ); } }; validate_lg_config_k(value, function_name) diff --git a/crates/sail-function/src/aggregate/kurtosis.rs b/crates/sail-function/src/aggregate/kurtosis.rs index 72085d74e..8f22d6564 100644 --- a/crates/sail-function/src/aggregate/kurtosis.rs +++ b/crates/sail-function/src/aggregate/kurtosis.rs @@ -9,8 +9,8 @@ use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; use datafusion::scalar::ScalarValue; use datafusion_common::types::{ - logical_float16, logical_float32, logical_float64, logical_int16, logical_int32, logical_int64, - logical_int8, logical_uint16, logical_uint32, logical_uint64, logical_uint8, NativeType, + NativeType, logical_float16, logical_float32, logical_float64, logical_int8, logical_int16, + logical_int32, logical_int64, logical_uint8, logical_uint16, logical_uint32, logical_uint64, }; use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; diff --git a/crates/sail-function/src/aggregate/max_min_by.rs b/crates/sail-function/src/aggregate/max_min_by.rs index 5ec37a5b3..826d00ee8 100644 --- a/crates/sail-function/src/aggregate/max_min_by.rs +++ b/crates/sail-function/src/aggregate/max_min_by.rs @@ -12,7 +12,7 @@ use datafusion::logical_expr::expr::{AggregateFunction, Sort}; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::simplify::SimplifyContext; use datafusion::logical_expr::utils::format_state_name; -use datafusion::logical_expr::{function, Accumulator, AggregateUDFImpl, Signature, Volatility}; +use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility, function}; use datafusion::prelude::Expr; #[derive(Debug)] diff --git a/crates/sail-function/src/aggregate/mod.rs b/crates/sail-function/src/aggregate/mod.rs index 8347efad0..1ac61e29d 100644 --- a/crates/sail-function/src/aggregate/mod.rs +++ b/crates/sail-function/src/aggregate/mod.rs @@ -15,7 +15,7 @@ /// let scalar = match_string_type!(DataType::Utf8View, Some("hello".to_string()))?; /// ``` macro_rules! match_string_type { - ($data_type:expr, $value:expr) => { + ($data_type:expr_2021, $value:expr_2021) => { match $data_type { DataType::Utf8View => Ok(ScalarValue::Utf8View($value)), DataType::LargeUtf8 => Ok(ScalarValue::LargeUtf8($value)), @@ -40,7 +40,7 @@ macro_rules! match_string_type { /// let scalar = interval_none!(IntervalUnit::YearMonth)?; /// ``` macro_rules! interval_none { - ($unit:expr) => { + ($unit:expr_2021) => { match $unit { datafusion::arrow::datatypes::IntervalUnit::YearMonth => { Ok(ScalarValue::IntervalYearMonth(None)) @@ -57,7 +57,7 @@ macro_rules! interval_none { /// Macro to handle Duration type matching for None values macro_rules! duration_none { - ($unit:expr) => { + ($unit:expr_2021) => { match $unit { datafusion::arrow::datatypes::TimeUnit::Second => Ok(ScalarValue::DurationSecond(None)), datafusion::arrow::datatypes::TimeUnit::Millisecond => { diff --git a/crates/sail-function/src/aggregate/mode.rs b/crates/sail-function/src/aggregate/mode.rs index d80e22616..3809b8a15 100644 --- a/crates/sail-function/src/aggregate/mode.rs +++ b/crates/sail-function/src/aggregate/mode.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::fmt::Debug; use std::hash::Hash; use std::sync::Arc; @@ -9,13 +8,13 @@ use datafusion::arrow::array::{ }; use datafusion::arrow::datatypes::{ DataType, Date32Type, Date64Type, Field, FieldRef, Float16Type, Float32Type, Float64Type, - Int16Type, Int32Type, Int64Type, Int8Type, Time32MillisecondType, Time32SecondType, + Int8Type, Int16Type, Int32Type, Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType, - TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, - UInt64Type, UInt8Type, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, + UInt32Type, UInt64Type, }; use datafusion::common::cast::{as_list_array, as_primitive_array, as_string_array}; -use datafusion::common::{exec_err, not_impl_err}; +use datafusion::common::{HashMap, exec_err, not_impl_err}; use datafusion::error::{DataFusionError, Result}; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; diff --git a/crates/sail-function/src/aggregate/percentile.rs b/crates/sail-function/src/aggregate/percentile.rs index 25385edd6..c74c4e033 100644 --- a/crates/sail-function/src/aggregate/percentile.rs +++ b/crates/sail-function/src/aggregate/percentile.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -6,7 +5,7 @@ use datafusion::arrow; use datafusion::arrow::array::{Array, ArrayRef, AsArray, RecordBatch, RecordBatchOptions}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::common::cast::{as_float64_array, as_string_array}; -use datafusion::common::ScalarValue; +use datafusion::common::{HashMap, ScalarValue}; use datafusion::error::Result; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; @@ -102,54 +101,57 @@ impl AggregateUDFImpl for PercentileFunction { })?; // Try to extract as array of percentiles first - if let Ok(percentiles) = extract_percentiles_array(expr) { - // Multiple percentiles - create MultiPercentileAccumulator - // Validate all percentiles are within [0.0, 1.0] - for percentile in &percentiles { - if !(0.0..=1.0).contains(percentile) { + match extract_percentiles_array(expr) { + Ok(percentiles) => { + // Multiple percentiles - create MultiPercentileAccumulator + // Validate all percentiles are within [0.0, 1.0] + for percentile in &percentiles { + if !(0.0..=1.0).contains(percentile) { + return Err(DataFusionError::Execution(format!( + "Percentile value {} is out of range [0.0, 1.0]", + percentile + ))); + } + } + + match data_type { + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => Ok(Box::new( + MultiStringPercentileAccumulator::new(percentiles, data_type.clone()), + )), + DataType::Interval(_) | DataType::Duration(_) => Ok(Box::new( + MultiIntervalPercentileAccumulator::new(percentiles, data_type.clone()), + )), + _ => Ok(Box::new(MultiNumericPercentileAccumulator::new( + percentiles, + ))), + } + } + _ => { + // Single percentile - existing behavior + let percentile: f64 = extract_literal(expr).map_err(|e| { + DataFusionError::Execution(format!( + "Failed to extract percentile value in percentile(): {}", + e + )) + })?; + + // Validate that percentile is within [0.0, 1.0] + if !(0.0..=1.0).contains(&percentile) { return Err(DataFusionError::Execution(format!( "Percentile value {} is out of range [0.0, 1.0]", percentile ))); } - } - - match data_type { - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => Ok(Box::new( - MultiStringPercentileAccumulator::new(percentiles, data_type.clone()), - )), - DataType::Interval(_) | DataType::Duration(_) => Ok(Box::new( - MultiIntervalPercentileAccumulator::new(percentiles, data_type.clone()), - )), - _ => Ok(Box::new(MultiNumericPercentileAccumulator::new( - percentiles, - ))), - } - } else { - // Single percentile - existing behavior - let percentile: f64 = extract_literal(expr).map_err(|e| { - DataFusionError::Execution(format!( - "Failed to extract percentile value in percentile(): {}", - e - )) - })?; - // Validate that percentile is within [0.0, 1.0] - if !(0.0..=1.0).contains(&percentile) { - return Err(DataFusionError::Execution(format!( - "Percentile value {} is out of range [0.0, 1.0]", - percentile - ))); - } - - match data_type { - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => Ok(Box::new( - StringPercentileAccumulator::new(percentile, data_type.clone()), - )), - DataType::Interval(_) | DataType::Duration(_) => Ok(Box::new( - IntervalPercentileAccumulator::new(percentile, data_type.clone()), - )), - _ => Ok(Box::new(NumericPercentileAccumulator::new(percentile))), + match data_type { + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => Ok(Box::new( + StringPercentileAccumulator::new(percentile, data_type.clone()), + )), + DataType::Interval(_) | DataType::Duration(_) => Ok(Box::new( + IntervalPercentileAccumulator::new(percentile, data_type.clone()), + )), + _ => Ok(Box::new(NumericPercentileAccumulator::new(percentile))), + } } } } @@ -1362,7 +1364,7 @@ impl Accumulator for MultiIntervalPercentileAccumulator { return Err(DataFusionError::Execution(format!( "Unexpected data type for interval percentile: {:?}", self.data_type - ))) + ))); } }; @@ -1416,7 +1418,7 @@ impl Accumulator for MultiIntervalPercentileAccumulator { return Err(DataFusionError::Execution(format!( "Unexpected data type: {:?}", self.data_type - ))) + ))); } }; results.push(null_scalar); @@ -1532,7 +1534,7 @@ impl Accumulator for MultiIntervalPercentileAccumulator { return Err(DataFusionError::Execution(format!( "Unexpected data type for interval percentile: {:?}", self.data_type - ))) + ))); } }; @@ -1656,7 +1658,7 @@ fn scalar_to_f64(scalar: &ScalarValue) -> Result { return Err(DataFusionError::Execution(format!( "Cannot convert percentile literal {:?} to f64", scalar - ))) + ))); } }; Ok(percentile) diff --git a/crates/sail-function/src/aggregate/percentile_disc.rs b/crates/sail-function/src/aggregate/percentile_disc.rs index a39cfec1e..f32886186 100644 --- a/crates/sail-function/src/aggregate/percentile_disc.rs +++ b/crates/sail-function/src/aggregate/percentile_disc.rs @@ -3,28 +3,29 @@ use std::mem::{size_of, size_of_val}; use std::sync::Arc; use datafusion::arrow::array::{ - Array, ArrayRef, ArrowNumericType, AsArray, ListArray, PrimitiveArray, + Array, ArrayRef, ArrowNumericType, AsArray, Float64Array, Int64Array, ListArray, PrimitiveArray, }; use datafusion::arrow::buffer::{OffsetBuffer, ScalarBuffer}; use datafusion::arrow::datatypes::{ DataType, Decimal128Type, Decimal256Type, Field, FieldRef, Float16Type, Float32Type, - Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, - UInt8Type, + Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalUnit, UInt8Type, UInt16Type, + UInt32Type, UInt64Type, }; -use datafusion::common::{DataFusionError, HashSet, Result, ScalarValue}; +use datafusion::common::{DataFusionError, Result, ScalarValue}; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::utils::format_state_name; use datafusion::logical_expr::{ - Accumulator, AggregateUDFImpl, GroupsAccumulator, Signature, TypeSignature, Volatility, + Accumulator, AggregateUDFImpl, GroupsAccumulator, Signature, Volatility, }; -use crate::aggregate::percentile_disc_groups::{ - DistinctPercentileDiscAccumulator, PercentileDiscGroupsAccumulator, +use crate::aggregate::percentile_disc_groups::PercentileDiscGroupsAccumulator; +use crate::aggregate::utils::{ + calculate_percentile_disc, cast_to_type_with_safe, extract_percentile_literal, + extract_percentiles_array, percentile_disc_index, }; -use crate::aggregate::utils::{calculate_percentile_disc, cast_to_type, validate_percentile}; macro_rules! dispatch_numeric_type { - ($input_dt:expr, $helper:ident, $err_msg:expr) => { + ($input_dt:expr_2021, $helper:ident, $err_msg:expr_2021) => { match &$input_dt { DataType::Int8 => $helper!(Int8Type, $input_dt), DataType::Int16 => $helper!(Int16Type, $input_dt), @@ -50,53 +51,73 @@ macro_rules! dispatch_numeric_type { #[derive(Debug, PartialEq, Eq, Hash)] pub struct PercentileDisc { signature: Signature, + /// When `true`, casting a non-numeric STRING ORDER BY column to `f64` is + /// strict (invalid values raise an error). When `false`, the cast is safe + /// (invalid values become NULL and are ignored). Mirrors + /// `spark.sql.ansi.enabled`. + ansi_mode: bool, } impl Default for PercentileDisc { fn default() -> Self { - Self::new() + Self::new(false) } } impl PercentileDisc { - pub fn new() -> Self { - let numeric_types = [ - DataType::Int8, - DataType::Int16, - DataType::Int32, - DataType::Int64, - DataType::UInt8, - DataType::UInt16, - DataType::UInt32, - DataType::UInt64, - DataType::Float16, - DataType::Float32, - DataType::Float64, - ]; - let mut variants = Vec::with_capacity(numeric_types.len()); - for num in &numeric_types { - variants.push(TypeSignature::Exact(vec![num.clone(), DataType::Float64])); - } + pub fn new(ansi_mode: bool) -> Self { + // `user_defined` so the planner calls our `coerce_types` (see there). Self { - signature: Signature::one_of(variants, Volatility::Immutable), + signature: Signature::user_defined(Volatility::Immutable), + ansi_mode, } } - /// Extracts percentile value and adjusts for descending order. - fn resolve_percentile(args: &AccumulatorArgs) -> Result { - let percentile = validate_percentile(&args.exprs[1])?; + /// `true` when running under `spark.sql.ansi.enabled = true`. + pub fn ansi_mode(&self) -> bool { + self.ansi_mode + } - let is_descending = args - .order_bys + /// `true` when the (single) `ORDER BY` direction is `DESC`. + fn is_descending(args: &AccumulatorArgs) -> bool { + args.order_bys .first() .map(|sort_expr| sort_expr.options.descending) - .unwrap_or(false); + .unwrap_or(false) + } - Ok(if is_descending { - 1.0 - percentile - } else { - percentile - }) + /// Spark requires exactly one `ORDER BY` expression inside `WITHIN GROUP` + /// (`INVALID_WITHIN_GROUP_EXPRESSION.WRONG_NUM_ORDERINGS`). The discrete + /// percentile is defined over a single ordered dimension, so reject anything + /// else rather than silently using only the first expression. + fn ensure_single_order_by(args: &AccumulatorArgs) -> Result<()> { + if args.order_bys.len() != 1 { + return Err(DataFusionError::Plan(format!( + "percentile_disc requires exactly one ORDER BY expression within \ + the WITHIN GROUP clause, got {}", + args.order_bys.len() + ))); + } + Ok(()) + } + + /// Extract the scalar percentile literal. We do NOT invert for `DESC` here: + /// `percentile_disc`'s discrete index is asymmetric, so the descending + /// adjustment is applied inside `calculate_percentile_disc` instead. + fn resolve_percentile(args: &AccumulatorArgs) -> Result { + extract_percentile_literal(&args.exprs[1]) + } + + /// Returns `Some(percentiles)` if `args.exprs[1]` is a list literal of + /// percentile values, `None` if it's a scalar. + fn try_resolve_percentiles_array(args: &AccumulatorArgs) -> Result>> { + // `coerce_types` normalizes any list flavor to `List(Float64)`, so we + // only need to check for `List`. + if !matches!(args.exprs[1].data_type(args.schema)?, DataType::List(_)) { + return Ok(None); + } + let percentiles = extract_percentiles_array(&args.exprs[1])?; + Ok(Some(percentiles)) } } @@ -109,71 +130,291 @@ impl AggregateUDFImpl for PercentileDisc { &self.signature } + fn supports_within_group_clause(&self) -> bool { + // `percentile_disc` is an ordered-set aggregate (`WITHIN GROUP`). + // Tell DataFusion so it can format the schema name correctly + // (`WITHIN GROUP (ORDER BY ...)` instead of `ORDER BY ...`) and skip + // the duplicated leading arg when rendering. + true + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + if arg_types.len() != 2 { + return Err(DataFusionError::Plan(format!( + "percentile_disc expects 2 arguments, got {}", + arg_types.len() + ))); + } + // arg[0]: ORDER BY column. Spark accepts every numeric type and STRING + // (cast-to-double semantics, always returns `double`) and ANSI day-time + // intervals (returns the interval type). Numerics route through `Float64`; + // STRING and INTERVAL/DURATION are left UNCHANGED so the accumulator can + // handle them (string→double cast with the ANSI flag; intervals stored as + // `i64` and reconstructed to the original type). + let order_by = match &arg_types[0] { + dt if dt.is_numeric() => DataType::Float64, + // Leave STRING types UNCHANGED so the planner inserts no cast. The + // string→double cast is performed inside the accumulator with a + // safe/strict flag driven by ANSI mode: under ANSI a non-numeric + // string errors, otherwise it becomes NULL and is ignored. Forcing + // `Float64` here would make DataFusion insert a strict planning-time + // cast that fails before the accumulator runs. + dt @ (DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View) => dt.clone(), + // Calendar intervals (month-day-nano, e.g. `make_interval`) are NOT + // orderable in Spark — reject them like Spark's INVALID_ORDERING_TYPE. + DataType::Interval(IntervalUnit::MonthDayNano) => { + return Err(DataFusionError::Plan( + "percentile_disc: calendar INTERVAL (month-day-nano) is not orderable; \ + use an ANSI day-time interval" + .into(), + )); + } + // Keep ANSI day-time/year-month INTERVAL and DURATION so the discrete + // value (and its type) survives to the accumulator; Spark returns the + // same interval type. + dt @ (DataType::Interval(_) | DataType::Duration(_)) => dt.clone(), + other => { + return Err(DataFusionError::Plan(format!( + "percentile_disc: ORDER BY column must be numeric, string, or interval, got {other}" + ))); + } + }; + // arg[1]: percentile literal — either a single value (scalar form) or + // a `List`/`LargeList`/`FixedSizeList` (array form). Coerce to + // `Float64` / `List` respectively. The accepted-type + // matrix mirrors what Spark validates at planning time: + // + // scalar array element + // numeric ✓ ✓ + // STRING (Utf8 etc.) ✓ (lazy) ✗ + // `Null` (untyped) ✗ ✓ (empty `array()`) + // bool/date/timestamp/binary/etc. ✗ ✗ + // + // STRING is permitted only for the scalar form because Spark accepts + // it via implicit cast there but rejects `array` at planning. + // `Null` element is permitted for the array form because that is the + // type of an empty `array()` literal (and of `array(NULL)`, which + // Spark also accepts treating NULL as `0.0`). + let percentile = match &arg_types[1] { + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => { + let elem = field.data_type(); + if !elem.is_numeric() && !matches!(elem, DataType::Null) { + return Err(DataFusionError::Plan(format!( + "percentile_disc: percentile array elements must be numeric, got {elem}" + ))); + } + DataType::List(Arc::new(Field::new("item", DataType::Float64, true))) + } + dt if dt.is_numeric() => DataType::Float64, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Float64, + other => { + return Err(DataFusionError::Plan(format!( + "percentile_disc: percentile must be numeric, got {other}" + ))); + } + }; + Ok(vec![order_by, percentile]) + } + fn return_type(&self, arg_types: &[DataType]) -> Result { + if matches!(arg_types.get(1), Some(DataType::List(_))) { + // INTERVAL/DURATION inputs yield `array`; everything else + // yields `array`. + let item = if matches!(arg_types[0], DataType::Interval(_) | DataType::Duration(_)) { + arg_types[0].clone() + } else { + DataType::Float64 + }; + return Ok(DataType::List(Arc::new(Field::new("item", item, true)))); + } + // STRING ORDER BY columns are left unchanged by `coerce_types` (no + // planning-time cast), but Spark's `percentile_disc` always returns + // `double` for string input, so report `Float64` here. + if matches!( + arg_types[0], + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) { + return Ok(DataType::Float64); + } Ok(arg_types[0].clone()) } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - let input_type = args.input_fields[0].data_type().clone(); - let field = Field::new_list_field(input_type, true); + let is_array_percentiles = matches!( + args.input_fields.get(1).map(|f| f.data_type()), + Some(DataType::List(_)) + ); + let input_type = args.input_fields[0].data_type(); + let value_type = if matches!(input_type, DataType::Interval(_) | DataType::Duration(_)) { + // Intervals are stored as their `i64` representation (scalar and array forms). + DataType::Int64 + } else if is_array_percentiles { + DataType::Float64 + } else if matches!( + input_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) { + // STRING input is left uncoerced at plan time but stored as `f64` + // inside the accumulator. + DataType::Float64 + } else { + input_type.clone() + }; + let field = Field::new_list_field(value_type, true); let state_name = if args.is_distinct { "distinct_percentile_disc" } else { "percentile_disc" }; - Ok(vec![Field::new( - format_state_name(args.name, state_name), - DataType::List(Arc::new(field)), - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, state_name), + DataType::List(Arc::new(field)), + true, + ) + .into(), + ]) } fn accumulator(&self, args: AccumulatorArgs) -> Result> { + // Spark rejects `DISTINCT` on `WITHIN GROUP` aggregates with + // `INVALID_WITHIN_GROUP_EXPRESSION.DISTINCT_UNSUPPORTED`. The DISTINCT + // keyword is meaningless here (the function's value is a position in + // the sorted population, not a count of distinct elements) — surface a + // matching error rather than silently de-duplicate the input. + if args.is_distinct { + return Err(DataFusionError::Plan( + "percentile_disc does not support DISTINCT with WITHIN GROUP".into(), + )); + } + Self::ensure_single_order_by(&args)?; + + let ansi_mode = self.ansi_mode; + + // Array-of-percentiles form: `percentile_disc(array(p1, p2, ...))`. + if let Some(percentiles) = Self::try_resolve_percentiles_array(&args)? { + let descending = Self::is_descending(&args); + let input_dt = args.exprs[0].data_type(args.schema)?; + // INTERVAL/DURATION array form returns `array`, not doubles. + if matches!(input_dt, DataType::Interval(_) | DataType::Duration(_)) { + return Ok(Box::new(MultiIntervalPercentileDiscAccumulator { + all_values: vec![], + percentiles, + descending, + data_type: input_dt, + })); + } + return Ok(Box::new(MultiPercentileDiscAccumulator { + all_values: vec![], + percentiles, + descending, + ansi_mode, + })); + } + + // Scalar percentile form: `percentile_disc(p)`. let percentile = Self::resolve_percentile(&args)?; + let descending = Self::is_descending(&args); + // DISTINCT was rejected above, so there's only one accumulator shape. macro_rules! helper { - ($t:ty, $dt:expr) => { - if args.is_distinct { - Ok(Box::new(DistinctPercentileDiscAccumulator::<$t> { - data_type: $dt.clone(), - distinct_values: HashSet::default(), - percentile, - })) - } else { - Ok(Box::new(PercentileDiscAccumulator::<$t> { - data_type: $dt.clone(), - all_values: vec![], - percentile, - })) - } + ($t:ty, $dt:expr_2021) => { + Ok(Box::new(PercentileDiscAccumulator::<$t> { + data_type: $dt.clone(), + all_values: vec![], + percentile, + descending, + ansi_mode, + })) }; } let input_dt = args.exprs[0].data_type(args.schema)?; + // STRING input is stored as `f64`; the string→double cast happens in + // `update_batch` with a safe/strict flag driven by ANSI mode. + if matches!( + input_dt, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) { + return Ok(Box::new(PercentileDiscAccumulator:: { + data_type: DataType::Float64, + all_values: vec![], + percentile, + descending, + ansi_mode, + })); + } + // INTERVAL/DURATION input: store the `i64` representation and reconstruct + // the interval type on output (Spark returns the interval type). + if matches!(input_dt, DataType::Interval(_) | DataType::Duration(_)) { + return Ok(Box::new(IntervalPercentileDiscAccumulator { + all_values: vec![], + percentile, + descending, + data_type: input_dt.clone(), + })); + } dispatch_numeric_type!(input_dt, helper, "PercentileDiscAccumulator") } fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool { - !args.is_distinct + if args.is_distinct { + return false; + } + // Array-of-percentiles and INTERVAL/DURATION inputs are not supported by + // the groups accumulator fast path; fall back to the regular per-group + // accumulator above (which handles both). + let arr_form = args + .exprs + .get(1) + .and_then(|e| e.data_type(args.schema).ok()) + .map(|dt| matches!(dt, DataType::List(_))) + .unwrap_or(false); + let interval_input = args + .exprs + .first() + .and_then(|e| e.data_type(args.schema).ok()) + .map(|dt| matches!(dt, DataType::Interval(_) | DataType::Duration(_))) + .unwrap_or(false); + !arr_form && !interval_input } fn create_groups_accumulator( &self, args: AccumulatorArgs, ) -> Result> { + Self::ensure_single_order_by(&args)?; let percentile = Self::resolve_percentile(&args)?; + let descending = Self::is_descending(&args); + let ansi_mode = self.ansi_mode; macro_rules! helper { - ($t:ty, $dt:expr) => { + ($t:ty, $dt:expr_2021) => { Ok(Box::new(PercentileDiscGroupsAccumulator::<$t>::new( - $dt, percentile, + $dt, percentile, descending, ansi_mode, ))) }; } let input_dt = args.exprs[0].data_type(args.schema)?; + // STRING input is stored as `f64`; the string→double cast happens in + // `update_batch` with a safe/strict flag driven by ANSI mode. + if matches!( + input_dt, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) { + return Ok(Box::new( + PercentileDiscGroupsAccumulator::::new( + DataType::Float64, + percentile, + descending, + ansi_mode, + ), + )); + } dispatch_numeric_type!(input_dt, helper, "PercentileDiscGroupsAccumulator") } } @@ -182,14 +423,19 @@ struct PercentileDiscAccumulator { data_type: DataType, all_values: Vec, percentile: f64, + descending: bool, + /// When `true`, the input→storage cast is strict (invalid string values + /// error). When `false`, it is safe (invalid values become NULL and are + /// ignored). Mirrors `spark.sql.ansi.enabled`. + ansi_mode: bool, } impl Debug for PercentileDiscAccumulator { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "PercentileDiscAccumulator({}, percentile={})", - self.data_type, self.percentile + "PercentileDiscAccumulator({}, percentile={}, descending={})", + self.data_type, self.percentile, self.descending ) } } @@ -215,7 +461,7 @@ impl Accumulator for PercentileDiscAccumulator { } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - let values_array = cast_to_type(&values[0], &self.data_type)?; + let values_array = cast_to_type_with_safe(&values[0], &self.data_type, !self.ansi_mode)?; let null_count = values_array.null_count(); let values = values_array.as_primitive::(); @@ -234,7 +480,7 @@ impl Accumulator for PercentileDiscAccumulator { fn evaluate(&mut self) -> Result { let d = std::mem::take(&mut self.all_values); - let value = calculate_percentile_disc::(d, self.percentile); + let value = calculate_percentile_disc::(d, self.percentile, self.descending); ScalarValue::new_primitive::(value, &self.data_type) } @@ -243,8 +489,355 @@ impl Accumulator for PercentileDiscAccumulator { } } -pub fn percentile_disc_udaf() -> Arc { +/// Accumulator for the array-of-percentiles form: stores values as `f64` +/// and computes each percentile from the same buffer. +/// +/// Output is `List` matching Spark's `array` return type. +struct MultiPercentileDiscAccumulator { + all_values: Vec, + percentiles: Vec, + descending: bool, + /// When `true`, the input→`f64` cast is strict (invalid string values + /// error). When `false`, it is safe (invalid values become NULL and are + /// ignored). Mirrors `spark.sql.ansi.enabled`. + ansi_mode: bool, +} + +impl Debug for MultiPercentileDiscAccumulator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "MultiPercentileDiscAccumulator(percentiles={:?}, len={}, descending={})", + self.percentiles, + self.all_values.len(), + self.descending + ) + } +} + +impl Accumulator for MultiPercentileDiscAccumulator { + fn state(&mut self) -> Result> { + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![ + 0_i32, + self.all_values.len() as i32, + ])); + let values_array = Float64Array::new( + ScalarBuffer::from(std::mem::take(&mut self.all_values)), + None, + ); + let list_array = ListArray::new( + Arc::new(Field::new_list_field(DataType::Float64, true)), + offsets, + Arc::new(values_array), + None, + ); + Ok(vec![ScalarValue::List(Arc::new(list_array))]) + } + + fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { + let values_array = cast_to_type_with_safe(&values[0], &DataType::Float64, !self.ansi_mode)?; + let null_count = values_array.null_count(); + let values = values_array.as_primitive::(); + self.all_values.reserve(values.len() - null_count); + self.all_values.extend(values.iter().flatten()); + Ok(()) + } + + fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { + let array = states[0].as_list::(); + for v in array.iter().flatten() { + self.update_batch(&[v])?; + } + Ok(()) + } + + fn evaluate(&mut self) -> Result { + // When the input is empty OR the percentile array is empty (e.g. + // callers passed `array()`), return a NULL list — matches Spark, which + // collapses `percentile_disc(array()) WITHIN GROUP (ORDER BY x)` to + // NULL rather than an empty `array`. + if self.all_values.is_empty() || self.percentiles.is_empty() { + let field = Arc::new(Field::new_list_field(DataType::Float64, true)); + return Ok(ScalarValue::List(Arc::new(ListArray::new_null(field, 1)))); + } + // Sort once (O(n log n)) and index into the sorted slice for each + // percentile (O(1) per pick). This avoids cloning the full value + // buffer for every requested percentile, which was O(k·n) for `k` + // percentiles and `n` values. f64 uses total_cmp to keep NaN ordering + // consistent with `calculate_percentile_disc`'s ArrowNativeTypeOp. + let mut sorted = std::mem::take(&mut self.all_values); + sorted.sort_unstable_by(|a, b| a.total_cmp(b)); + let len = sorted.len(); + let results: Vec> = self + .percentiles + .iter() + .map(|&p| Some(sorted[percentile_disc_index(len, p, self.descending)])) + .collect(); + let values_array = Float64Array::from_iter(results); + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, values_array.len() as i32])); + let list_array = ListArray::new( + Arc::new(Field::new_list_field(DataType::Float64, true)), + offsets, + Arc::new(values_array), + None, + ); + Ok(ScalarValue::List(Arc::new(list_array))) + } + + fn size(&self) -> usize { + size_of_val(self) + + self.all_values.capacity() * size_of::() + + self.percentiles.capacity() * size_of::() + } +} + +/// Accumulator for ANSI interval / duration ORDER BY columns. Each value is +/// stored as its `i64` representation; on output we pick the discrete percentile +/// and reconstruct the original interval/duration type. Mirrors `percentile.rs`'s +/// interval handling but selects discretely (no interpolation). +struct IntervalPercentileDiscAccumulator { + all_values: Vec, + percentile: f64, + descending: bool, + data_type: DataType, +} + +impl Debug for IntervalPercentileDiscAccumulator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "IntervalPercentileDiscAccumulator({}, percentile={}, descending={})", + self.data_type, self.percentile, self.descending + ) + } +} + +/// Convert one interval/duration array element to its scalar `i64` representation. +fn interval_to_i64(data_type: &DataType, array: &ArrayRef, row: usize) -> Result { + use datafusion::arrow::datatypes::{IntervalUnit, TimeUnit}; + let v = match data_type { + DataType::Interval(IntervalUnit::YearMonth) => array + .as_primitive::() + .value(row) as i64, + DataType::Interval(IntervalUnit::DayTime) => { + let val = array + .as_primitive::() + .value(row); + val.days as i64 * 86_400_000 + val.milliseconds as i64 + } + DataType::Interval(IntervalUnit::MonthDayNano) => { + let val = array + .as_primitive::() + .value(row); + val.months as i64 * 2_592_000_000_000_000 + + val.days as i64 * 86_400_000_000_000 + + val.nanoseconds + } + DataType::Duration(TimeUnit::Second) => array + .as_primitive::() + .value(row), + DataType::Duration(TimeUnit::Millisecond) => array + .as_primitive::() + .value(row), + DataType::Duration(TimeUnit::Microsecond) => array + .as_primitive::() + .value(row), + DataType::Duration(TimeUnit::Nanosecond) => array + .as_primitive::() + .value(row), + other => { + return Err(DataFusionError::Internal(format!( + "percentile_disc interval accumulator does not support {other}" + ))); + } + }; + Ok(v) +} + +/// Reconstruct a `ScalarValue` of `data_type` from an `i64`, or a typed NULL. +fn i64_to_interval_scalar(data_type: &DataType, value: Option) -> Result { + use datafusion::arrow::datatypes::{ + IntervalDayTime, IntervalMonthDayNano, IntervalUnit, TimeUnit, + }; + Ok(match data_type { + DataType::Interval(IntervalUnit::YearMonth) => { + ScalarValue::IntervalYearMonth(value.map(|v| v as i32)) + } + DataType::Interval(IntervalUnit::DayTime) => { + ScalarValue::IntervalDayTime(value.map(|v| IntervalDayTime { + days: (v / 86_400_000) as i32, + milliseconds: (v % 86_400_000) as i32, + })) + } + DataType::Interval(IntervalUnit::MonthDayNano) => { + ScalarValue::IntervalMonthDayNano(value.map(|v| { + let months = (v / 2_592_000_000_000_000) as i32; + let rem = v % 2_592_000_000_000_000; + IntervalMonthDayNano { + months, + days: (rem / 86_400_000_000_000) as i32, + nanoseconds: rem % 86_400_000_000_000, + } + })) + } + DataType::Duration(TimeUnit::Second) => ScalarValue::DurationSecond(value), + DataType::Duration(TimeUnit::Millisecond) => ScalarValue::DurationMillisecond(value), + DataType::Duration(TimeUnit::Microsecond) => ScalarValue::DurationMicrosecond(value), + DataType::Duration(TimeUnit::Nanosecond) => ScalarValue::DurationNanosecond(value), + other => { + return Err(DataFusionError::Internal(format!( + "percentile_disc interval accumulator does not support {other}" + ))); + } + }) +} + +impl Accumulator for IntervalPercentileDiscAccumulator { + fn state(&mut self) -> Result> { + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![ + 0_i32, + self.all_values.len() as i32, + ])); + let values_array = Int64Array::from(std::mem::take(&mut self.all_values)); + let list_array = ListArray::new( + Arc::new(Field::new_list_field(DataType::Int64, true)), + offsets, + Arc::new(values_array), + None, + ); + Ok(vec![ScalarValue::List(Arc::new(list_array))]) + } + + fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { + let array = &values[0]; + self.all_values.reserve(array.len() - array.null_count()); + for row in 0..array.len() { + if !array.is_null(row) { + let v = interval_to_i64(&self.data_type, array, row)?; + self.all_values.push(v); + } + } + Ok(()) + } + + fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { + let array = states[0].as_list::(); + for v in array.iter().flatten() { + self.all_values + .extend(v.as_primitive::().iter().flatten()); + } + Ok(()) + } + + fn evaluate(&mut self) -> Result { + if self.all_values.is_empty() { + return i64_to_interval_scalar(&self.data_type, None); + } + let mut sorted = std::mem::take(&mut self.all_values); + sorted.sort_unstable(); + let idx = percentile_disc_index(sorted.len(), self.percentile, self.descending); + i64_to_interval_scalar(&self.data_type, Some(sorted[idx])) + } + + fn size(&self) -> usize { + size_of_val(self) + self.all_values.capacity() * size_of::() + } +} + +/// Array-of-percentiles form for interval/duration inputs. Stores values as `i64`, +/// computes each percentile discretely, and reconstructs a `List` +/// (matching Spark's `array` return, not `array`). +struct MultiIntervalPercentileDiscAccumulator { + all_values: Vec, + percentiles: Vec, + descending: bool, + data_type: DataType, +} + +impl Debug for MultiIntervalPercentileDiscAccumulator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "MultiIntervalPercentileDiscAccumulator({}, percentiles={:?}, descending={})", + self.data_type, self.percentiles, self.descending + ) + } +} + +impl Accumulator for MultiIntervalPercentileDiscAccumulator { + fn state(&mut self) -> Result> { + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![ + 0_i32, + self.all_values.len() as i32, + ])); + let values_array = Int64Array::from(std::mem::take(&mut self.all_values)); + let list_array = ListArray::new( + Arc::new(Field::new_list_field(DataType::Int64, true)), + offsets, + Arc::new(values_array), + None, + ); + Ok(vec![ScalarValue::List(Arc::new(list_array))]) + } + + fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { + let array = &values[0]; + self.all_values.reserve(array.len() - array.null_count()); + for row in 0..array.len() { + if !array.is_null(row) { + let v = interval_to_i64(&self.data_type, array, row)?; + self.all_values.push(v); + } + } + Ok(()) + } + + fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { + let array = states[0].as_list::(); + for v in array.iter().flatten() { + self.all_values + .extend(v.as_primitive::().iter().flatten()); + } + Ok(()) + } + + fn evaluate(&mut self) -> Result { + // Empty input or empty percentile array → NULL list (matches the numeric form). + if self.all_values.is_empty() || self.percentiles.is_empty() { + let field = Arc::new(Field::new_list_field(self.data_type.clone(), true)); + return Ok(ScalarValue::List(Arc::new(ListArray::new_null(field, 1)))); + } + let mut sorted = std::mem::take(&mut self.all_values); + sorted.sort_unstable(); + let len = sorted.len(); + let scalars: Vec = self + .percentiles + .iter() + .map(|&p| { + let idx = percentile_disc_index(len, p, self.descending); + i64_to_interval_scalar(&self.data_type, Some(sorted[idx])) + }) + .collect::>>()?; + let values_array = ScalarValue::iter_to_array(scalars)?; + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, values_array.len() as i32])); + let list_array = ListArray::new( + Arc::new(Field::new_list_field(self.data_type.clone(), true)), + offsets, + values_array, + None, + ); + Ok(ScalarValue::List(Arc::new(list_array))) + } + + fn size(&self) -> usize { + size_of_val(self) + + self.all_values.capacity() * size_of::() + + self.percentiles.capacity() * size_of::() + } +} + +pub fn percentile_disc_udaf(ansi_mode: bool) -> Arc { Arc::new(datafusion::logical_expr::AggregateUDF::from( - PercentileDisc::new(), + PercentileDisc::new(ansi_mode), )) } diff --git a/crates/sail-function/src/aggregate/percentile_disc_groups.rs b/crates/sail-function/src/aggregate/percentile_disc_groups.rs index 8e1d82a73..18bad54a4 100644 --- a/crates/sail-function/src/aggregate/percentile_disc_groups.rs +++ b/crates/sail-function/src/aggregate/percentile_disc_groups.rs @@ -1,4 +1,4 @@ -use std::fmt::{Debug, Formatter}; +use std::fmt::Debug; use std::mem::{size_of, size_of_val}; use std::sync::Arc; @@ -8,25 +8,33 @@ use datafusion::arrow::array::{ }; use datafusion::arrow::buffer::{OffsetBuffer, ScalarBuffer}; use datafusion::arrow::datatypes::{DataType, Field}; -use datafusion::common::{DataFusionError, HashSet, Result, ScalarValue}; -use datafusion::logical_expr::{Accumulator, EmitTo, GroupsAccumulator}; -use datafusion::physical_expr::aggregate::utils::Hashable; +use datafusion::common::{DataFusionError, Result}; +use datafusion::logical_expr::{EmitTo, GroupsAccumulator}; -use crate::aggregate::utils::{calculate_percentile_disc, cast_to_type, filtered_null_mask}; +use crate::aggregate::utils::{ + calculate_percentile_disc, cast_to_type_with_safe, filtered_null_mask, +}; #[derive(Debug)] pub struct PercentileDiscGroupsAccumulator { data_type: DataType, group_values: Vec>, percentile: f64, + descending: bool, + /// When `true`, casts of the ORDER BY input to the storage type are strict + /// (invalid values raise an error). When `false`, casts are safe (invalid + /// values become NULL and are ignored). Mirrors `spark.sql.ansi.enabled`. + ansi_mode: bool, } impl PercentileDiscGroupsAccumulator { - pub fn new(data_type: DataType, percentile: f64) -> Self { + pub fn new(data_type: DataType, percentile: f64, descending: bool, ansi_mode: bool) -> Self { Self { data_type, group_values: Vec::new(), percentile, + descending, + ansi_mode, } } @@ -93,7 +101,7 @@ impl GroupsAccumulator for PercentileDiscGroupsAccum opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { - let values_array = cast_to_type(&values[0], &self.data_type)?; + let values_array = cast_to_type_with_safe(&values[0], &self.data_type, !self.ansi_mode)?; let values = values_array.as_primitive::(); self.group_values.resize(total_num_groups, Vec::new()); @@ -129,10 +137,10 @@ impl GroupsAccumulator for PercentileDiscGroupsAccum .zip(group_indices.iter()) .zip(values.iter()) .for_each(|((filter_value, &group_index), new_value)| { - if let Some(true) = filter_value { - if let Some(new_value) = new_value { - self.group_values[group_index].push(new_value) - } + if let Some(true) = filter_value + && let Some(new_value) = new_value + { + self.group_values[group_index].push(new_value) } }) } @@ -198,7 +206,7 @@ impl GroupsAccumulator for PercentileDiscGroupsAccum let mut evaluate_result_builder = PrimitiveBuilder::::new().with_data_type(self.data_type.clone()); for values in emit_group_values { - let value = calculate_percentile_disc::(values, self.percentile); + let value = calculate_percentile_disc::(values, self.percentile, self.descending); evaluate_result_builder.append_option(value); } @@ -210,7 +218,7 @@ impl GroupsAccumulator for PercentileDiscGroupsAccum values: &[ArrayRef], opt_filter: Option<&BooleanArray>, ) -> Result> { - let values_array = cast_to_type(&values[0], &self.data_type)?; + let values_array = cast_to_type_with_safe(&values[0], &self.data_type, !self.ansi_mode)?; let input_array = values_array.as_primitive::(); let values = PrimitiveArray::::new(input_array.values().clone(), None) @@ -250,73 +258,3 @@ impl GroupsAccumulator for PercentileDiscGroupsAccum + self.group_values.capacity() * size_of::>() } } - -pub struct DistinctPercentileDiscAccumulator { - pub data_type: DataType, - pub distinct_values: HashSet>, - pub percentile: f64, -} - -impl Debug for DistinctPercentileDiscAccumulator { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "DistinctPercentileDiscAccumulator({}, percentile={})", - self.data_type, self.percentile - ) - } -} - -impl Accumulator for DistinctPercentileDiscAccumulator { - fn state(&mut self) -> Result> { - let all_values = self - .distinct_values - .iter() - .map(|x| ScalarValue::new_primitive::(Some(x.0), &self.data_type)) - .collect::>>()?; - - let arr = ScalarValue::new_list_nullable(&all_values, &self.data_type); - Ok(vec![ScalarValue::List(arr)]) - } - - fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - if values.is_empty() { - return Ok(()); - } - - let values = cast_to_type(&values[0], &self.data_type)?; - let array = values.as_primitive::(); - match array.nulls().filter(|x| x.null_count() > 0) { - Some(n) => { - for idx in n.valid_indices() { - self.distinct_values.insert(Hashable(array.value(idx))); - } - } - None => array.values().iter().for_each(|x| { - self.distinct_values.insert(Hashable(*x)); - }), - } - Ok(()) - } - - fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { - let array = states[0].as_list::(); - for v in array.iter().flatten() { - self.update_batch(&[v])? - } - Ok(()) - } - - fn evaluate(&mut self) -> Result { - let d = std::mem::take(&mut self.distinct_values) - .into_iter() - .map(|v| v.0) - .collect::>(); - let value = calculate_percentile_disc::(d, self.percentile); - ScalarValue::new_primitive::(value, &self.data_type) - } - - fn size(&self) -> usize { - std::mem::size_of_val(self) + self.distinct_values.capacity() * size_of::() - } -} diff --git a/crates/sail-function/src/aggregate/product.rs b/crates/sail-function/src/aggregate/product.rs index 0f232f38d..a946ec017 100644 --- a/crates/sail-function/src/aggregate/product.rs +++ b/crates/sail-function/src/aggregate/product.rs @@ -9,8 +9,8 @@ use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; use datafusion::scalar::ScalarValue; use datafusion_common::types::{ - logical_float16, logical_float32, logical_float64, logical_int16, logical_int32, logical_int64, - logical_int8, logical_uint16, logical_uint32, logical_uint64, logical_uint8, NativeType, + NativeType, logical_float16, logical_float32, logical_float64, logical_int8, logical_int16, + logical_int32, logical_int64, logical_uint8, logical_uint16, logical_uint32, logical_uint64, }; use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; diff --git a/crates/sail-function/src/aggregate/schema_of_variant_agg.rs b/crates/sail-function/src/aggregate/schema_of_variant_agg.rs index 6d0126390..050ff91ee 100644 --- a/crates/sail-function/src/aggregate/schema_of_variant_agg.rs +++ b/crates/sail-function/src/aggregate/schema_of_variant_agg.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeMap; use std::sync::Arc; use arrow::array::{Array, ArrayRef}; @@ -7,10 +6,13 @@ use datafusion::common::Result; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; use datafusion::scalar::ScalarValue; -use parquet_variant::Variant; use parquet_variant_compute::VariantArray; -use crate::scalar::variant::spark_schema_of_variant::variant_to_spark_type; +use crate::scalar::variant::spark_schema_of_variant::{ + merge_variant_types, variant_to_inferred_type, variant_type_from_spark_type, + variant_type_to_spark_type, +}; +use crate::schema_inference::InferredType; /// Aggregate function that merges variant schemas across rows. /// @@ -68,185 +70,9 @@ impl AggregateUDFImpl for SchemaOfVariantAggFunction { } } -/// Internal representation of a merged variant schema. -#[derive(Debug, Clone, PartialEq)] -enum MergedType { - /// A concrete primitive type (e.g., "BIGINT", "STRING", "BOOLEAN") - Primitive(String), - /// VOID (from null values) - Void, - /// An object with merged fields (field_name → merged_type) - Object(BTreeMap), - /// An array with a merged element type - Array(Box), - /// Mixed types that cannot be unified → VARIANT - Variant, -} - -impl MergedType { - /// Parse a Spark type string back into a MergedType. - fn from_spark_type(s: &str) -> Result { - if s == "VOID" { - return Ok(MergedType::Void); - } - if s == "VARIANT" { - return Ok(MergedType::Variant); - } - if let Some(inner) = s.strip_prefix("OBJECT<").and_then(|s| s.strip_suffix('>')) { - if inner.is_empty() { - return Ok(MergedType::Object(BTreeMap::new())); - } - let mut fields = BTreeMap::new(); - for field_str in split_top_level(inner) { - let (name, type_str) = field_str.split_once(": ").ok_or_else(|| { - datafusion::common::DataFusionError::Execution(format!( - "schema_of_variant_agg: invalid object field entry '{field_str}'" - )) - })?; - fields.insert( - name.trim().to_string(), - MergedType::from_spark_type(type_str.trim())?, - ); - } - return Ok(MergedType::Object(fields)); - } - if let Some(inner) = s.strip_prefix("ARRAY<").and_then(|s| s.strip_suffix('>')) { - return Ok(MergedType::Array(Box::new(MergedType::from_spark_type( - inner, - )?))); - } - Ok(MergedType::Primitive(s.to_string())) - } - - /// Convert to Spark type string. - fn to_spark_type(&self) -> String { - match self { - MergedType::Void => "VOID".to_string(), - MergedType::Primitive(s) => s.clone(), - MergedType::Variant => "VARIANT".to_string(), - MergedType::Object(fields) => { - if fields.is_empty() { - return "OBJECT<>".to_string(); - } - let fields_str: Vec = fields - .iter() - .map(|(name, ty)| format!("{name}: {}", ty.to_spark_type())) - .collect(); - format!("OBJECT<{}>", fields_str.join(", ")) - } - MergedType::Array(elem) => { - format!("ARRAY<{}>", elem.to_spark_type()) - } - } - } - - /// Estimate heap memory usage recursively. - fn estimated_size(&self) -> usize { - match self { - MergedType::Void | MergedType::Variant => 0, - MergedType::Primitive(s) => s.len(), - MergedType::Object(fields) => fields - .iter() - .map(|(k, v)| { - k.len() + v.estimated_size() + std::mem::size_of::<(String, MergedType)>() - }) - .sum(), - MergedType::Array(elem) => std::mem::size_of::() + elem.estimated_size(), - } - } - - /// Merge two types together. - fn merge(self, other: MergedType) -> MergedType { - match (self, other) { - // VOID merges with anything → the other type - (MergedType::Void, other) | (other, MergedType::Void) => other, - // VARIANT absorbs everything - (MergedType::Variant, _) | (_, MergedType::Variant) => MergedType::Variant, - // Same primitive → keep - (MergedType::Primitive(a), MergedType::Primitive(b)) if a == b => { - MergedType::Primitive(a) - } - // Different primitives → VARIANT - (MergedType::Primitive(_), MergedType::Primitive(_)) => MergedType::Variant, - // Object + Object → merge fields - (MergedType::Object(mut a), MergedType::Object(b)) => { - for (key, val) in b { - match a.entry(key) { - std::collections::btree_map::Entry::Occupied(mut entry) => { - let old = std::mem::replace(entry.get_mut(), MergedType::Void); - *entry.get_mut() = old.merge(val); - } - std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(val); - } - } - } - MergedType::Object(a) - } - // Array + Array → merge element types - (MergedType::Array(a), MergedType::Array(b)) => { - MergedType::Array(Box::new(a.merge(*b))) - } - // Incompatible types → VARIANT - _ => MergedType::Variant, - } - } -} - -/// Split a string by top-level commas (respecting nested angle brackets and parentheses). -/// Handles types like `DECIMAL(10,2)` inside `OBJECT<...>` fields. -fn split_top_level(s: &str) -> Vec<&str> { - let mut results = Vec::new(); - let mut angle_depth: usize = 0; - let mut paren_depth: usize = 0; - let mut start = 0; - for (i, ch) in s.char_indices() { - match ch { - '<' => angle_depth += 1, - '>' => angle_depth = angle_depth.saturating_sub(1), - '(' => paren_depth += 1, - ')' => paren_depth = paren_depth.saturating_sub(1), - ',' if angle_depth == 0 && paren_depth == 0 => { - results.push(s[start..i].trim()); - start = i + 1; - } - _ => {} - } - } - if start < s.len() { - results.push(s[start..].trim()); - } - results -} - -/// Convert a Variant to a MergedType. -fn variant_to_merged_type(variant: &Variant) -> MergedType { - match variant { - Variant::Null => MergedType::Void, - Variant::Object(obj) => { - let mut fields = BTreeMap::new(); - for (name, value) in obj.iter() { - fields.insert(name.to_string(), variant_to_merged_type(&value)); - } - MergedType::Object(fields) - } - Variant::List(list) => { - let mut elem_type = MergedType::Void; - for elem in list.iter() { - elem_type = elem_type.merge(variant_to_merged_type(&elem)); - } - MergedType::Array(Box::new(elem_type)) - } - _ => { - // Use the same type string as schema_of_variant for primitives - MergedType::Primitive(variant_to_spark_type(variant)) - } - } -} - #[derive(Debug)] struct SchemaOfVariantAggAccumulator { - merged_schema: Option, + merged_schema: Option, } impl Accumulator for SchemaOfVariantAggAccumulator { @@ -255,9 +81,9 @@ impl Accumulator for SchemaOfVariantAggAccumulator { let variant_array = VariantArray::try_new(arr.as_ref())?; for variant in variant_array.iter().flatten() { - let current_type = variant_to_merged_type(&variant); + let current_type = variant_to_inferred_type(&variant); self.merged_schema = Some(match self.merged_schema.take() { - Some(existing) => existing.merge(current_type), + Some(existing) => merge_variant_types(existing, current_type), None => current_type, }); } @@ -268,7 +94,7 @@ impl Accumulator for SchemaOfVariantAggAccumulator { let schema_str = self .merged_schema .as_ref() - .map_or_else(|| "VOID".to_string(), |t| t.to_spark_type()); + .map_or_else(|| "VOID".to_string(), variant_type_to_spark_type); Ok(vec![ScalarValue::Utf8(Some(schema_str))]) } @@ -286,9 +112,9 @@ impl Accumulator for SchemaOfVariantAggAccumulator { if schema_arr.is_null(i) { continue; } - let other_schema = MergedType::from_spark_type(schema_arr.value(i))?; + let other_schema = variant_type_from_spark_type(schema_arr.value(i))?; self.merged_schema = Some(match self.merged_schema.take() { - Some(existing) => existing.merge(other_schema), + Some(existing) => merge_variant_types(existing, other_schema), None => other_schema, }); } @@ -300,7 +126,7 @@ impl Accumulator for SchemaOfVariantAggAccumulator { let result = self .merged_schema .as_ref() - .map_or_else(|| "VOID".to_string(), |t| t.to_spark_type()); + .map_or_else(|| "VOID".to_string(), variant_type_to_spark_type); Ok(ScalarValue::Utf8(Some(result))) } diff --git a/crates/sail-function/src/aggregate/skewness.rs b/crates/sail-function/src/aggregate/skewness.rs index f7a7b0319..097a23ba2 100644 --- a/crates/sail-function/src/aggregate/skewness.rs +++ b/crates/sail-function/src/aggregate/skewness.rs @@ -2,12 +2,12 @@ use std::fmt::Debug; use datafusion::arrow::array::{ArrayRef, AsArray, Float64Array}; use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Float64Type}; -use datafusion::common::{downcast_value, Result, ScalarValue}; +use datafusion::common::{Result, ScalarValue, downcast_value}; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; use datafusion_common::types::{ - logical_float16, logical_float32, logical_float64, logical_int16, logical_int32, logical_int64, - logical_int8, logical_uint16, logical_uint32, logical_uint64, logical_uint8, NativeType, + NativeType, logical_float16, logical_float32, logical_float64, logical_int8, logical_int16, + logical_int32, logical_int64, logical_uint8, logical_uint16, logical_uint32, logical_uint64, }; use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; diff --git a/crates/sail-function/src/aggregate/theta_sketch.rs b/crates/sail-function/src/aggregate/theta_sketch.rs index a05e8ff8c..dfff6ea5b 100644 --- a/crates/sail-function/src/aggregate/theta_sketch.rs +++ b/crates/sail-function/src/aggregate/theta_sketch.rs @@ -7,13 +7,13 @@ use datafusion::logical_expr::utils::format_state_name; use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, Signature, TypeSignature, Volatility, }; -use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err}; use super::utils::get_scalar_value; use crate::theta_sketch::{ - compact_update_sketch_bytes, empty_compact_sketch_bytes, intersect_sketch_bytes, - new_update_sketch, normalize_sketch_bytes, union_sketches, update_sketch_from_array, - validate_lg_nom_entries, DEFAULT_LG_NOM_ENTRIES, + DEFAULT_LG_NOM_ENTRIES, compact_update_sketch_bytes, empty_compact_sketch_bytes, + intersect_sketch_bytes, new_update_sketch, normalize_sketch_bytes, union_sketches, + update_sketch_from_array, validate_lg_nom_entries, }; #[derive(Debug, PartialEq, Eq, Hash)] @@ -58,12 +58,14 @@ impl AggregateUDFImpl for ThetaSketchAggFunction { } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - Ok(vec![Field::new( - format_state_name(args.name, "theta_sketch"), - DataType::Binary, - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, "theta_sketch"), + DataType::Binary, + true, + ) + .into(), + ]) } } @@ -109,12 +111,14 @@ impl AggregateUDFImpl for ThetaUnionAggFunction { } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - Ok(vec![Field::new( - format_state_name(args.name, "theta_union"), - DataType::Binary, - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, "theta_union"), + DataType::Binary, + true, + ) + .into(), + ]) } } @@ -156,12 +160,14 @@ impl AggregateUDFImpl for ThetaIntersectionAggFunction { } fn state_fields(&self, args: StateFieldsArgs) -> Result> { - Ok(vec![Field::new( - format_state_name(args.name, "theta_intersection"), - DataType::Binary, - true, - ) - .into()]) + Ok(vec![ + Field::new( + format_state_name(args.name, "theta_intersection"), + DataType::Binary, + true, + ) + .into(), + ]) } } @@ -414,7 +420,7 @@ fn resolve_lg_nom_entries(args: &AccumulatorArgs, index: usize, function_name: & return exec_err!( "{function_name} requires lgNomEntries to be a non-null integer literal, got {}", value.data_type() - ) + ); } }; validate_lg_nom_entries(value, function_name) diff --git a/crates/sail-function/src/aggregate/try_avg.rs b/crates/sail-function/src/aggregate/try_avg.rs index 5dbe76bd1..c1c910a62 100644 --- a/crates/sail-function/src/aggregate/try_avg.rs +++ b/crates/sail-function/src/aggregate/try_avg.rs @@ -8,10 +8,10 @@ use datafusion::arrow::datatypes::{ DataType, Decimal128Type, Field, FieldRef, Int64Type, IntervalUnit, IntervalYearMonthType, }; use datafusion_common::arrow::datatypes::Float64Type; -use datafusion_common::{downcast_value, DataFusionError, ScalarValue}; +use datafusion_common::{DataFusionError, ScalarValue, downcast_value}; +use datafusion_expr::AggregateUDFImpl; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; -use datafusion_expr::AggregateUDFImpl; use datafusion_expr_common::accumulator::Accumulator; use datafusion_expr_common::signature::{Signature, Volatility}; @@ -141,11 +141,11 @@ impl TryAvgAccumulator { } }, }; - if let Some(sum) = self.sum_dec128 { - if exceeds_decimal128_precision(sum, p) { - self.failed = true; - return; - } + if let Some(sum) = self.sum_dec128 + && exceeds_decimal128_precision(sum, p) + { + self.failed = true; + return; } self.inc_count(); if self.failed { @@ -238,12 +238,11 @@ impl TryAvgAccumulator { total.checked_sub(half) }; - if let Some(adj) = adjusted { - if let Some(avg) = adj.checked_div(denom) { - if fits_i32(avg) { - return ScalarValue::IntervalYearMonth(Some(avg as i32)); - } - } + if let Some(adj) = adjusted + && let Some(avg) = adj.checked_div(denom) + && fits_i32(avg) + { + return ScalarValue::IntervalYearMonth(Some(avg as i32)); } ScalarValue::IntervalYearMonth(None) } else { @@ -289,7 +288,7 @@ impl Accumulator for TryAvgAccumulator { ref dt => { return Err(DataFusionError::Execution(format!( "try_avg: type not supported update_batch: {dt:?}" - ))) + ))); } } Ok(()) @@ -425,7 +424,7 @@ impl Accumulator for TryAvgAccumulator { ref dt => { return Err(DataFusionError::Execution(format!( "try_avg: type not supported in merge_batch: {dt:?}" - ))) + ))); } } @@ -475,7 +474,7 @@ impl AggregateUDFImpl for TryAvgFunction { ref other => { return Err(DataFusionError::Plan(format!( "try_avg: unsupported type: {other:?}" - ))) + ))); } }; @@ -528,7 +527,7 @@ impl AggregateUDFImpl for TryAvgFunction { other => { return Err(DataFusionError::Plan(format!( "try_avg: unsupported type: {other:?}" - ))) + ))); } }; Ok(vec![coerced]) @@ -545,8 +544,8 @@ mod tests { use datafusion::arrow::array::{Int32Array, Int64Array}; use datafusion::arrow::datatypes::IntervalUnit; - use datafusion_common::arrow::array::Float64Array; use datafusion_common::ScalarValue; + use datafusion_common::arrow::array::Float64Array; use super::*; diff --git a/crates/sail-function/src/aggregate/utils.rs b/crates/sail-function/src/aggregate/utils.rs index a60c2b316..e4ed4969d 100644 --- a/crates/sail-function/src/aggregate/utils.rs +++ b/crates/sail-function/src/aggregate/utils.rs @@ -2,12 +2,15 @@ use std::sync::Arc; use datafusion::arrow::array::{ Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, BooleanArray, RecordBatch, + RecordBatchOptions, }; use datafusion::arrow::buffer::NullBuffer; -use datafusion::arrow::datatypes::{DataType, Schema}; +use datafusion::arrow::compute::CastOptions; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::common::{DataFusionError, Result, ScalarValue}; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; +use sail_common_datafusion::literal::LiteralValue; /// /// This is part of the shared contract for bitmap aggregate functions that @@ -15,9 +18,31 @@ use datafusion::physical_expr::PhysicalExpr; pub(crate) const BITMAP_NUM_BYTES: usize = 4 * 1024; /// Casts an array to the target type if it doesn't already match. +/// +/// Uses a safe cast (invalid values become NULL). pub fn cast_to_type(array: &ArrayRef, target_type: &DataType) -> Result { + cast_to_type_with_safe(array, target_type, true) +} + +/// Casts an array to the target type if it doesn't already match, choosing +/// between a safe cast (`safe = true`, invalid values become NULL) and a strict +/// cast (`safe = false`, invalid values raise an error). +/// +/// This drives the ANSI-mode behavior of percentile aggregates: under +/// `spark.sql.ansi.enabled = false` a non-numeric string is coerced to NULL and +/// ignored, while under ANSI mode an invalid cast surfaces an error. +pub fn cast_to_type_with_safe( + array: &ArrayRef, + target_type: &DataType, + safe: bool, +) -> Result { if array.data_type() != target_type { - datafusion::arrow::compute::cast(array, target_type).map_err(DataFusionError::from) + let options = CastOptions { + safe, + ..Default::default() + }; + datafusion::arrow::compute::cast_with_options(array, target_type, &options) + .map_err(DataFusionError::from) } else { Ok(Arc::clone(array)) } @@ -26,12 +51,11 @@ pub fn cast_to_type(array: &ArrayRef, target_type: &DataType) -> Result) -> Result { let empty_schema = Arc::new(Schema::empty()); let batch = RecordBatch::new_empty(Arc::clone(&empty_schema)); - if let ColumnarValue::Scalar(s) = expr.evaluate(&batch)? { - Ok(s) - } else { - Err(DataFusionError::Internal( + match expr.evaluate(&batch)? { + ColumnarValue::Scalar(s) => Ok(s), + _ => Err(DataFusionError::Internal( "Didn't expect ColumnarValue::Array".to_string(), - )) + )), } } @@ -49,7 +73,7 @@ pub fn validate_percentile(expr: &Arc) -> Result { return Err(DataFusionError::Plan(format!( "Percentile value for 'PERCENTILE_DISC' must be Float32 or Float64 literal (got data type {})", sv.data_type() - ))) + ))); } }; @@ -73,24 +97,148 @@ pub fn filtered_null_mask( NullBuffer::union(opt_filter.as_ref(), input.nulls()) } -/// Returns the value at the given discrete percentile. +/// Convert a `ScalarValue` to `f64`, supporting floats, integers, and decimals +/// (`Decimal32`/`Decimal64`/`Decimal128`). /// -/// Returns an actual value from the dataset (no interpolation). Specifically, -/// returns the first value whose cumulative distribution is >= the requested percentile. -pub fn calculate_percentile_disc( - mut values: Vec, - percentile: f64, -) -> Option { - let len = values.len(); - if len == 0 { - return None; +/// Used for extracting percentile literals from physical expressions where the +/// literal may arrive as any numeric scalar. +pub(crate) fn scalar_to_f64(scalar: &ScalarValue) -> Result { + let percentile: f64 = match scalar { + ScalarValue::Float64(Some(v)) => *v, + ScalarValue::Float32(Some(v)) => *v as f64, + ScalarValue::Float16(Some(v)) => f64::from(*v), + ScalarValue::Int8(_) + | ScalarValue::Int16(_) + | ScalarValue::Int32(_) + | ScalarValue::Int64(_) + | ScalarValue::UInt8(_) + | ScalarValue::UInt16(_) + | ScalarValue::UInt32(_) + | ScalarValue::UInt64(_) => { + let int_val = LiteralValue(scalar).try_to_i64().map_err(|e| { + DataFusionError::Plan(format!( + "Cannot convert percentile literal {scalar:?} to integer: {e}" + )) + })?; + int_val as f64 + } + ScalarValue::Decimal32(Some(v), _precision, scale) => { + (*v as f64) / 10f64.powi(*scale as i32) + } + ScalarValue::Decimal64(Some(v), _precision, scale) => { + (*v as f64) / 10f64.powi(*scale as i32) + } + ScalarValue::Decimal128(Some(v), _precision, scale) => { + (*v as f64) / 10f64.powi(*scale as i32) + } + _ => { + return Err(DataFusionError::Plan(format!( + "Cannot convert percentile literal {scalar:?} to f64" + ))); + } + }; + Ok(percentile) +} + +/// Extract a single `f64` percentile from a physical expression literal. +/// +/// Validates that the value is within `[0.0, 1.0]`. Errors with +/// `DataFusionError::Plan` for non-literal arguments and out-of-range values — +/// both are detectable at planning time. +pub(crate) fn extract_percentile_literal(expr: &Arc) -> Result { + let scalar = evaluate_percentile_literal(expr)?; + if matches!(scalar, ScalarValue::List(_)) { + return Err(DataFusionError::Plan( + "expected a scalar percentile literal, got a list".into(), + )); } + let percentile = scalar_to_f64(&scalar)?; + if !(0.0..=1.0).contains(&percentile) { + return Err(DataFusionError::Plan(format!( + "Percentile value {percentile} is out of range [0.0, 1.0]" + ))); + } + Ok(percentile) +} - let cmp = |x: &T::Native, y: &T::Native| x.compare(*y); +/// Extract a vector of `f64` percentiles from a physical expression that +/// evaluates to a `List` of numeric scalars. +pub(crate) fn extract_percentiles_array(expr: &Arc) -> Result> { + let scalar = evaluate_percentile_literal(expr)?; + match scalar { + ScalarValue::List(arr) => { + // An empty array literal yields an empty Vec; callers decide what + // to do with it (Spark's `percentile_disc(array())` returns NULL). + let values = arr.values(); + let mut percentiles = Vec::with_capacity(values.len()); + for i in 0..values.len() { + // Spark treats NULL percentile-list elements as `0.0` (returns + // the minimum value), rather than erroring. Mirror that + // behavior so the WITHIN GROUP form aligns with Spark. + let percentile = if values.is_null(i) { + 0.0 + } else { + let val_scalar = ScalarValue::try_from_array(values.as_ref(), i)?; + scalar_to_f64(&val_scalar)? + }; + if !(0.0..=1.0).contains(&percentile) { + return Err(DataFusionError::Plan(format!( + "Percentile value {percentile} is out of range [0.0, 1.0]" + ))); + } + percentiles.push(percentile); + } + Ok(percentiles) + } + other => Err(DataFusionError::Plan(format!( + "Expected List type for percentiles array, got: {other:?}" + ))), + } +} - // percentile_disc: return actual value at the target index (no interpolation) - // index = ceil(percentile * len) - 1, clamped to valid range - let index = if percentile == 0.0 { +/// Evaluate `expr` as a constant percentile literal and return the scalar. +/// +/// Rejects non-literal expressions (e.g. column references) with +/// `DataFusionError::Plan` so the failure is detected at planning time with a +/// clear message, rather than as a low-level evaluation error against a +/// synthetic empty `RecordBatch`. +fn evaluate_percentile_literal(expr: &Arc) -> Result { + // Evaluate against an empty 1-row batch: any foldable constant (a literal, or + // a cast/expression over literals like `CAST(1 AS DOUBLE)`) yields a value, + // while a column-dependent expression errors because the batch has no columns. + let schema = Arc::new(Schema::new(Vec::::new())); + let batch = RecordBatch::try_new_with_options( + schema, + vec![], + &RecordBatchOptions::default().with_row_count(Some(1)), + )?; + let col_val = expr.evaluate(&batch)?; + Ok(match col_val { + ColumnarValue::Scalar(s) => s, + ColumnarValue::Array(arr) => ScalarValue::try_from_array(arr.as_ref(), 0)?, + }) +} + +/// ASC-sorted index for `percentile_disc(percentile)` over a population of +/// length `len`. `descending=true` returns the equivalent index for an +/// `ORDER BY ... DESC` selection performed on the same ASC-sorted slice. +/// +/// `percentile_disc`'s index formula is asymmetric, so the `1 - percentile` +/// shortcut used for `percentile_cont` does NOT work here (e.g. for +/// `[1,2,3,4]` with `p=0.25 DESC` the answer is `4`, not `3`). Callers must +/// pass a non-empty `len` and a percentile already validated to `[0.0, 1.0]`. +pub fn percentile_disc_index(len: usize, percentile: f64, descending: bool) -> usize { + debug_assert!(len > 0); + if descending { + if percentile == 0.0 { + len - 1 + } else if percentile == 1.0 { + 0 + } else { + len.saturating_sub((percentile * len as f64).ceil() as usize) + .min(len - 1) + } + } else if percentile == 0.0 { 0 } else if percentile == 1.0 { len - 1 @@ -98,8 +246,25 @@ pub fn calculate_percentile_disc( ((percentile * len as f64).ceil() as usize) .saturating_sub(1) .min(len - 1) - }; + } +} +/// Returns the value at the given discrete percentile. +/// +/// Returns an actual value from the dataset (no interpolation). Specifically, +/// returns the first value whose cumulative distribution is >= the requested +/// percentile. See [`percentile_disc_index`] for the DESC semantics. +pub fn calculate_percentile_disc( + mut values: Vec, + percentile: f64, + descending: bool, +) -> Option { + let len = values.len(); + if len == 0 { + return None; + } + let cmp = |x: &T::Native, y: &T::Native| x.compare(*y); + let index = percentile_disc_index(len, percentile, descending); // select_nth_unstable_by is O(n) partial sort - finds the value at index let (_, value, _) = values.select_nth_unstable_by(index, cmp); Some(*value) diff --git a/crates/sail-function/src/error.rs b/crates/sail-function/src/error.rs index 968d78df4..6a7cb45fd 100644 --- a/crates/sail-function/src/error.rs +++ b/crates/sail-function/src/error.rs @@ -1,6 +1,6 @@ use datafusion::arrow::datatypes::DataType; // TODO: https://github.com/apache/spark/tree/master/common/utils/src/main/resources/error -use datafusion_common::{exec_datafusion_err, internal_datafusion_err, DataFusionError}; +use datafusion_common::{DataFusionError, exec_datafusion_err, internal_datafusion_err}; pub fn invalid_arg_count_exec_err( function_name: &str, @@ -24,7 +24,9 @@ pub fn unsupported_data_type_exec_err( required: &str, provided: &DataType, ) -> DataFusionError { - exec_datafusion_err!("Unsupported Data Type: Spark `{function_name}` function expects {required}, got {provided}") + exec_datafusion_err!( + "Unsupported Data Type: Spark `{function_name}` function expects {required}, got {provided}" + ) } pub fn unsupported_data_types_exec_err( diff --git a/crates/sail-function/src/functions_nested_utils.rs b/crates/sail-function/src/functions_nested_utils.rs index 62a5a6776..e813d25ae 100644 --- a/crates/sail-function/src/functions_nested_utils.rs +++ b/crates/sail-function/src/functions_nested_utils.rs @@ -3,7 +3,7 @@ use datafusion::arrow::array::{Array, ArrayRef}; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::ColumnarValue; macro_rules! downcast_arg { - ($ARG:expr, $ARRAY_TYPE:ident) => {{ + ($ARG:expr_2021, $ARRAY_TYPE:ident) => {{ $ARG.as_any().downcast_ref::<$ARRAY_TYPE>().ok_or_else(|| { ::datafusion_common::DataFusionError::Internal(format!( "could not cast to {}", @@ -14,9 +14,7 @@ macro_rules! downcast_arg { } macro_rules! opt_downcast_arg { - ($ARG:expr, $ARRAY_TYPE:ident) => {{ - $ARG.as_any().downcast_ref::<$ARRAY_TYPE>() - }}; + ($ARG:expr_2021, $ARRAY_TYPE:ident) => {{ $ARG.as_any().downcast_ref::<$ARRAY_TYPE>() }}; } pub(crate) use downcast_arg; diff --git a/crates/sail-function/src/hll_sketch.rs b/crates/sail-function/src/hll_sketch.rs index 89a6d87f9..b7acf7aeb 100644 --- a/crates/sail-function/src/hll_sketch.rs +++ b/crates/sail-function/src/hll_sketch.rs @@ -1,9 +1,6 @@ -use arrow::array::{ - Array, ArrayRef, BinaryArray, Int32Array, Int64Array, LargeStringArray, StringArray, - StringViewArray, -}; -use arrow::datatypes::DataType; -use datafusion_common::{exec_err, DataFusionError, Result}; +use arrow::array::{Array, ArrayRef, AsArray}; +use arrow::datatypes::{DataType, Int32Type, Int64Type}; +use datafusion_common::{DataFusionError, Result, exec_err}; use datasketches::hash_value::{raw_bytes, sign_extend}; use datasketches::hll::{HllSketch, HllType, HllUnion}; @@ -46,8 +43,51 @@ pub(crate) fn update_hll_sketch_from_array( sketch: &mut HllSketch, values: &ArrayRef, ) -> Result<()> { - for row in 0..values.len() { - update_hll_sketch_from_array_value(sketch, values.as_ref(), row)?; + // Downcast once outside the row loop (the data type and concrete array are + // loop-invariant); `iter().flatten()` skips nulls without a per-row branch. + match values.data_type() { + DataType::Int32 => { + for v in values.as_primitive::().iter().flatten() { + sketch.update(sign_extend::from_i32(v)); + } + } + DataType::Int64 => { + for v in values.as_primitive::().iter().flatten() { + sketch.update(v); + } + } + DataType::Binary => { + for v in values.as_binary::().iter().flatten() { + if !v.is_empty() { + sketch.update(raw_bytes::from_slice(v)); + } + } + } + DataType::Utf8 => { + for v in values.as_string::().iter().flatten() { + if !v.is_empty() { + sketch.update(raw_bytes::from_str(v)); + } + } + } + DataType::LargeUtf8 => { + for v in values.as_string::().iter().flatten() { + if !v.is_empty() { + sketch.update(raw_bytes::from_str(v)); + } + } + } + DataType::Utf8View => { + for v in values.as_string_view().iter().flatten() { + if !v.is_empty() { + sketch.update(raw_bytes::from_str(v)); + } + } + } + DataType::Null => {} + data_type => { + return exec_err!("hll_sketch_agg does not support input type {data_type}"); + } } Ok(()) } @@ -109,90 +149,6 @@ where .unwrap_or_else(empty_hll_sketch_bytes)) } -fn update_hll_sketch_from_array_value( - sketch: &mut HllSketch, - values: &dyn Array, - row: usize, -) -> Result<()> { - if values.is_null(row) { - return Ok(()); - } - - match values.data_type() { - DataType::Int32 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("HLL sketch expected Int32Array".to_string()) - })?; - sketch.update(sign_extend::from_i32(values.value(row))); - } - DataType::Int64 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("HLL sketch expected Int64Array".to_string()) - })?; - sketch.update(values.value(row)); - } - DataType::Binary => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("HLL sketch expected BinaryArray".to_string()) - })?; - let value = values.value(row); - if !value.is_empty() { - sketch.update(raw_bytes::from_slice(value)); - } - } - DataType::Utf8 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("HLL sketch expected StringArray".to_string()) - })?; - let value = values.value(row); - if !value.is_empty() { - sketch.update(raw_bytes::from_str(value)); - } - } - DataType::LargeUtf8 => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("HLL sketch expected LargeStringArray".to_string()) - })?; - let value = values.value(row); - if !value.is_empty() { - sketch.update(raw_bytes::from_str(value)); - } - } - DataType::Utf8View => { - let values = values - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("HLL sketch expected StringViewArray".to_string()) - })?; - let value = values.value(row); - if !value.is_empty() { - sketch.update(raw_bytes::from_str(value)); - } - } - DataType::Null => {} - data_type => { - return exec_err!("hll_sketch_agg does not support input type {data_type}"); - } - } - Ok(()) -} - fn deserialize_hll_sketch(bytes: &[u8], function_name: &str) -> Result { HllSketch::deserialize(bytes).map_err(|error| { DataFusionError::Execution(format!( diff --git a/crates/sail-function/src/lib.rs b/crates/sail-function/src/lib.rs index b4ca0b0ce..507b248c0 100644 --- a/crates/sail-function/src/lib.rs +++ b/crates/sail-function/src/lib.rs @@ -4,6 +4,7 @@ mod functions_nested_utils; mod functions_utils; mod hll_sketch; pub mod scalar; +mod schema_inference; pub mod sketch { pub use crate::hll_sketch::DEFAULT_LG_CONFIG_K as DEFAULT_HLL_LG_CONFIG_K; pub use crate::theta_sketch::DEFAULT_LG_NOM_ENTRIES as DEFAULT_THETA_LG_NOM_ENTRIES; diff --git a/crates/sail-function/src/scalar/array/array_intersect.rs b/crates/sail-function/src/scalar/array/array_intersect.rs index 46294f399..474c6773d 100644 --- a/crates/sail-function/src/scalar/array/array_intersect.rs +++ b/crates/sail-function/src/scalar/array/array_intersect.rs @@ -1,12 +1,11 @@ // [CREDIT]: https://github.com/apache/datafusion/blob/4a41173ba3df9b5d47638599c819a1e6e46ad92b/datafusion/functions-nested/src/set_ops.rs // Adapted from DataFusion's set operation implementation with Spark-compatible empty array handling for array intersections. -use std::collections::HashSet; use std::sync::Arc; use datafusion::arrow::array::{ - new_empty_array, new_null_array, Array, ArrayRef, GenericListArray, OffsetSizeTrait, - UInt32Array, UInt64Array, + Array, ArrayRef, GenericListArray, OffsetSizeTrait, UInt32Array, UInt64Array, new_empty_array, + new_null_array, }; use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer}; use datafusion::arrow::compute::take; @@ -15,7 +14,7 @@ use datafusion::arrow::row::{RowConverter, SortField}; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::utils::take_function_args; use datafusion_common::{ - assert_eq_or_internal_err, exec_err, internal_err, DataFusionError, Result, + DataFusionError, HashSet, Result, assert_eq_or_internal_err, exec_err, internal_err, }; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -381,7 +380,7 @@ mod tests { use datafusion::arrow::buffer::OffsetBuffer; use datafusion::arrow::datatypes::{DataType, Field, Int32Type}; use datafusion_common::config::ConfigOptions; - use datafusion_common::{exec_err, Result}; + use datafusion_common::{Result, exec_err}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; use super::ArrayIntersect; diff --git a/crates/sail-function/src/scalar/array/array_position.rs b/crates/sail-function/src/scalar/array/array_position.rs index 8a834708e..9b519ced9 100644 --- a/crates/sail-function/src/scalar/array/array_position.rs +++ b/crates/sail-function/src/scalar/array/array_position.rs @@ -8,9 +8,9 @@ use arrow::array::{ use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::datatypes::{ ArrowPrimitiveType, DataType, Date32Type, Decimal128Type, Field, Float32Type, Float64Type, - Int16Type, Int32Type, Int64Type, Int8Type, TimestampMicrosecondType, + Int8Type, Int16Type, Int32Type, Int64Type, TimestampMicrosecondType, }; -use datafusion::common::{exec_err, DataFusionError, Result as DataFusionResult, ScalarValue}; +use datafusion::common::{DataFusionError, Result as DataFusionResult, ScalarValue, exec_err}; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -214,12 +214,12 @@ where let nulls = combined_nulls(list_array.nulls(), element.nulls()); let mut result = vec![0i64; num_rows]; - for (row_index, w) in offsets.windows(2).enumerate() { + for (row_index, &[start, end]) in offsets.array_windows::<2>().enumerate() { if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { continue; } - let start = w[0].as_usize(); - let end = w[1].as_usize(); + let start = start.as_usize(); + let end = end.as_usize(); let search_val = element_typed.value(row_index); for i in start..end { if !values_typed.is_null(i) && values_typed.value(i) == search_val { @@ -248,12 +248,12 @@ where let nulls = combined_nulls(list_array.nulls(), element.nulls()); let mut result = vec![0i64; num_rows]; - for (row_index, w) in offsets.windows(2).enumerate() { + for (row_index, &[start, end]) in offsets.array_windows::<2>().enumerate() { if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { continue; } - let start = w[0].as_usize(); - let end = w[1].as_usize(); + let start = start.as_usize(); + let end = end.as_usize(); let search_val = element_typed.value(row_index); let search_is_nan = search_val.is_nan(); for i in start..end { @@ -289,12 +289,12 @@ fn position_boolean( let nulls = combined_nulls(list_array.nulls(), element.nulls()); let mut result = vec![0i64; num_rows]; - for (row_index, w) in offsets.windows(2).enumerate() { + for (row_index, &[start, end]) in offsets.array_windows::<2>().enumerate() { if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { continue; } - let start = w[0].as_usize(); - let end = w[1].as_usize(); + let start = start.as_usize(); + let end = end.as_usize(); let search_val = element_typed.value(row_index); for i in start..end { if !values_typed.is_null(i) && values_typed.value(i) == search_val { @@ -320,12 +320,12 @@ fn position_string( let nulls = combined_nulls(list_array.nulls(), element.nulls()); let mut result = vec![0i64; num_rows]; - for (row_index, w) in offsets.windows(2).enumerate() { + for (row_index, &[start, end]) in offsets.array_windows::<2>().enumerate() { if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { continue; } - let start = w[0].as_usize(); - let end = w[1].as_usize(); + let start = start.as_usize(); + let end = end.as_usize(); let search_val = element_typed.value(row_index); for i in start..end { if !values_typed.is_null(i) && values_typed.value(i) == search_val { @@ -350,12 +350,12 @@ fn position_string_view( let nulls = combined_nulls(list_array.nulls(), element.nulls()); let mut result = vec![0i64; num_rows]; - for (row_index, w) in offsets.windows(2).enumerate() { + for (row_index, &[start, end]) in offsets.array_windows::<2>().enumerate() { if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { continue; } - let start = w[0].as_usize(); - let end = w[1].as_usize(); + let start = start.as_usize(); + let end = end.as_usize(); let search_val = element_typed.value(row_index); for i in start..end { if !values_typed.is_null(i) && values_typed.value(i) == search_val { @@ -379,12 +379,12 @@ fn position_fallback( let nulls = combined_nulls(list_array.nulls(), element.nulls()); let mut result = vec![0i64; num_rows]; - for (row_index, w) in offsets.windows(2).enumerate() { + for (row_index, &[start, end]) in offsets.array_windows::<2>().enumerate() { if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { continue; } - let start = w[0].as_usize(); - let end = w[1].as_usize(); + let start = start.as_usize(); + let end = end.as_usize(); let search_scalar = ScalarValue::try_from_array(element, row_index)?; for i in start..end { if !values.is_null(i) { diff --git a/crates/sail-function/src/scalar/array/arrays_zip.rs b/crates/sail-function/src/scalar/array/arrays_zip.rs index b7a98d8a0..d85067e2f 100644 --- a/crates/sail-function/src/scalar/array/arrays_zip.rs +++ b/crates/sail-function/src/scalar/array/arrays_zip.rs @@ -2,13 +2,13 @@ use std::ops::BitAnd; use std::sync::Arc; use datafusion::arrow::array::{ - new_empty_array, new_null_array, Array, ArrayRef, AsArray, FixedSizeListArray, - GenericListArray, NullArray, OffsetSizeTrait, StructArray, + Array, ArrayRef, AsArray, FixedSizeListArray, GenericListArray, OffsetSizeTrait, StructArray, + UInt64Array, new_null_array, }; use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer}; -use datafusion::arrow::compute::{cast, concat}; -use datafusion::arrow::datatypes::{DataType, Field, Fields}; -use datafusion_common::{arrow_err, exec_err, plan_err, DataFusionError, Result}; +use datafusion::arrow::compute::{cast, take}; +use datafusion::arrow::datatypes::{DataType, Field}; +use datafusion_common::{DataFusionError, Result, arrow_err, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; @@ -341,10 +341,15 @@ fn arrays_zip_generic( DataFusionError::Execution("`arrays_zip`: zero offset should always exist".to_string()) })?; - let mut struct_arrays = Vec::with_capacity(num_rows); + // Flatten build: compute the output offsets once and gather each struct column with a single + // `take`, null-padding ragged rows via null indices. Avoids building a `StructArray` per row + // and a final `concat` of all of them. + let n_args = lists.len(); let mut offsets = Vec::with_capacity(num_rows + 1); offsets.push(zero_offset); let mut last_offset = zero_offset; + let mut take_indices: Vec>> = (0..n_args).map(|_| Vec::new()).collect(); + let mut row_spans: Vec<(usize, usize)> = Vec::with_capacity(n_args); for row_idx in 0..num_rows { if validity_mask_opt @@ -355,78 +360,34 @@ fn arrays_zip_generic( continue; } - let mut arrays_one_row: Vec = Vec::with_capacity(lists.len()); - let mut lens_one_row: Vec = Vec::with_capacity(lists.len()); - let mut max_len_one_row = zero_offset; - let mut all_uniform = true; - for arg in &lists { - let arr = arg.value(row_idx); - let len = arg.value_length(row_idx); - if !lens_one_row.is_empty() && len != lens_one_row[0] { - all_uniform = false; - } - if len > max_len_one_row { - max_len_one_row = len; + row_spans.clear(); + let mut max_len = 0usize; + for list in &lists { + let start = list.value_offsets()[row_idx].as_usize(); + let len = list.value_length(row_idx).as_usize(); + row_spans.push((start, len)); + max_len = max_len.max(len); + } + for k in 0..max_len { + for (j, &(start, len)) in row_spans.iter().enumerate() { + take_indices[j].push((k < len).then(|| (start + k) as u64)); } - arrays_one_row.push(arr); - lens_one_row.push(len); } - - let arrays_padded = if all_uniform { - arrays_one_row - } else { - arrays_one_row - .iter() - .zip(lens_one_row.iter()) - .map(|(arr, len)| { - Ok(match (max_len_one_row - *len).as_usize() { - 0 => arr.clone(), - len_diff => Arc::new(concat(&[ - arr, - &cast(&NullArray::new(len_diff), arr.data_type())?, - ])?), - }) - }) - .collect::>>()? - }; - - let struct_array = to_struct_array( - arrays_padded.as_slice(), - field_names.as_slice(), - &arg_fields, - )?; - let offset = O::from_usize(struct_array.len()).ok_or_else(|| { - DataFusionError::Execution("`arrays_zip` offset overflow error".to_string()) - })?; - - last_offset += offset; + last_offset += O::usize_as(max_len); offsets.push(last_offset); - struct_arrays.push(struct_array); } - let values: ArrayRef = if struct_arrays.is_empty() { - // When all rows are null, struct_arrays is empty. Create an empty struct array. - let fields: Fields = field_names - .iter() - .zip(inner_fields.iter()) - .map(|(name, f)| { - Field::new(name, f.data_type().clone(), true).with_metadata(f.metadata().clone()) - }) - .collect(); - let empty_arrays: Vec = inner_fields - .iter() - .map(|f| new_empty_array(f.data_type())) - .collect(); - Arc::new(StructArray::try_new(fields, empty_arrays, None)?) - } else { - concat( - struct_arrays - .iter() - .map(|a| a.as_ref()) - .collect::>() - .as_slice(), - )? - }; + // One `take` per struct column; a `None` index null-pads a ragged position. + let columns = lists + .iter() + .zip(take_indices) + .map(|(list, indices)| { + let indices = UInt64Array::from(indices); + take(list.values().as_ref(), &indices, None).map_or_else(|err| arrow_err!(err), Ok) + }) + .collect::>>()?; + + let values = to_struct_array(columns.as_slice(), field_names.as_slice(), &arg_fields)?; Ok(Arc::new(GenericListArray::::try_new( struct_result_field(&inner_fields, &field_names), diff --git a/crates/sail-function/src/scalar/array/lambda_utils.rs b/crates/sail-function/src/scalar/array/lambda_utils.rs index 26a6b8259..1e59b3faf 100644 --- a/crates/sail-function/src/scalar/array/lambda_utils.rs +++ b/crates/sail-function/src/scalar/array/lambda_utils.rs @@ -10,12 +10,17 @@ /// (`index_array`/`offsets_to_indices`). use std::sync::Arc; -use datafusion::arrow::array::{Array, ArrayRef, AsArray, Int32Array, OffsetSizeTrait}; +use datafusion::arrow::array::{ + Array, ArrayRef, AsArray, BooleanArray, BooleanBuilder, Int32Array, OffsetSizeTrait, +}; use datafusion::arrow::buffer::OffsetBuffer; +use datafusion::arrow::compute::take_arrays; use datafusion::arrow::datatypes::DataType; -use datafusion_common::utils::{adjust_offsets_for_slice, list_values, take_function_args}; -use datafusion_common::{exec_err, plan_err, Result, ScalarValue}; -use datafusion_expr::{ColumnarValue, ValueOrLambda}; +use datafusion_common::utils::{ + adjust_offsets_for_slice, list_values, list_values_row_number, take_function_args, +}; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::{ColumnarValue, LambdaArgument, ValueOrLambda}; use crate::error::generic_exec_err; @@ -97,6 +102,102 @@ pub(crate) fn extract_list_values( Ok(ListValuesResult::Values(values)) } +/// Evaluates a boolean lambda element by element, stopping each row at the first +/// `stop_on` value, and reduces with Spark's three-valued logic. +/// +/// Spark's `ArrayExists`/`ArrayForAll` walk the elements in order and stop at the +/// first `true`/`false`, so an element past the stopping point is never evaluated +/// and can never raise. Evaluating the whole flattened batch at once — which is +/// what the vectorized path does — raises on elements Spark would have skipped. +/// This is the recovery path for exactly that case: it costs one lambda +/// evaluation per element, so callers must only reach it once the vectorized +/// evaluation has already failed. +pub(crate) fn short_circuit_boolean_reduce( + name: &str, + list_array: &ArrayRef, + values: &ArrayRef, + lambda: &LambdaArgument, + stop_on: bool, +) -> Result { + let (offsets, nulls) = match list_array.data_type() { + DataType::List(_) => { + let list = list_array.as_list::(); + ( + offsets_to_usize(&adjust_offsets_for_slice(list)), + list.nulls().cloned(), + ) + } + DataType::LargeList(_) => { + let list = list_array.as_list::(); + ( + offsets_to_usize(&adjust_offsets_for_slice(list)), + list.nulls().cloned(), + ) + } + other => return exec_err!("{name} expected list, got {other}"), + }; + + let row_numbers = list_values_row_number(list_array)?; + let num_rows = list_array.len(); + let mut builder = BooleanBuilder::with_capacity(num_rows); + + for row in 0..num_rows { + if nulls.as_ref().is_some_and(|n| n.is_null(row)) { + builder.append_null(); + continue; + } + let mut stopped = false; + let mut any_null = false; + for index in offsets[row]..offsets[row + 1] { + let element = values.slice(index, 1); + let element_param = || Ok(Arc::clone(&element)); + let params: [&dyn Fn() -> Result; 1] = [&element_param]; + let output = lambda.evaluate(¶ms, |arrays| { + Ok(take_arrays(arrays, &row_numbers.slice(index, 1), None)?) + })?; + match single_boolean(name, &output)? { + Some(value) if value == stop_on => { + stopped = true; + break; + } + Some(_) => {} + None => any_null = true, + } + } + if stopped { + builder.append_value(stop_on); + } else if any_null { + builder.append_null(); + } else { + builder.append_value(!stop_on); + } + } + + Ok(builder.finish()) +} + +fn offsets_to_usize(offsets: &OffsetBuffer) -> Vec { + offsets.iter().map(|offset| offset.as_usize()).collect() +} + +fn single_boolean(name: &str, output: &ColumnarValue) -> Result> { + if let ColumnarValue::Scalar(ScalarValue::Boolean(value)) = output { + return Ok(*value); + } + let array = output.clone().into_array(1)?; + let Some(array) = array.as_any().downcast_ref::() else { + return exec_err!( + "{name} lambda must return boolean, got {}", + array.data_type() + ); + }; + if array.is_null(0) { + Ok(None) + } else { + Ok(Some(array.value(0))) + } +} + /// 0-based per-sublist positions aligned with the flattened values of `list_array`. pub(crate) fn index_array(name: &str, list_array: &ArrayRef) -> Result { match list_array.data_type() { @@ -120,8 +221,8 @@ fn offsets_to_indices( .unwrap_or(0) .saturating_sub(offsets.first().map(|o| o.as_usize()).unwrap_or(0)); let mut out: Vec = Vec::with_capacity(total); - for w in offsets.windows(2) { - let len = w[1].as_usize() - w[0].as_usize(); + for &[start, end] in offsets.array_windows::<2>() { + let len = end.as_usize() - start.as_usize(); let len = i32::try_from(len) .map_err(|_| generic_exec_err(name, "array too large for Int32 index"))?; out.extend(0..len); diff --git a/crates/sail-function/src/scalar/array/spark_array.rs b/crates/sail-function/src/scalar/array/spark_array.rs index 7f93f1f29..71a1d4240 100644 --- a/crates/sail-function/src/scalar/array/spark_array.rs +++ b/crates/sail-function/src/scalar/array/spark_array.rs @@ -3,13 +3,13 @@ use std::sync::Arc; use datafusion::arrow::array::{ - make_array, new_empty_array, new_null_array, Array, ArrayData, ArrayRef, Capacities, - GenericListArray, MutableArrayData, NullArray, OffsetSizeTrait, + Array, ArrayData, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullArray, + OffsetSizeTrait, make_array, new_empty_array, new_null_array, }; use datafusion::arrow::buffer::OffsetBuffer; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::SingleRowListArrayBuilder; -use datafusion_common::{plan_datafusion_err, plan_err, Result}; +use datafusion_common::{Result, plan_datafusion_err, plan_err}; use datafusion_expr::type_coercion::binary::comparison_coercion; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, diff --git a/crates/sail-function/src/scalar/array/spark_array_aggregate.rs b/crates/sail-function/src/scalar/array/spark_array_aggregate.rs index 613f12e2d..9826c7741 100644 --- a/crates/sail-function/src/scalar/array/spark_array_aggregate.rs +++ b/crates/sail-function/src/scalar/array/spark_array_aggregate.rs @@ -8,14 +8,14 @@ use std::sync::Arc; use datafusion::arrow::array::{ - make_array, new_empty_array, new_null_array, Array, ArrayRef, AsArray, MutableArrayData, - OffsetSizeTrait, UInt64Array, + Array, ArrayRef, AsArray, MutableArrayData, OffsetSizeTrait, UInt64Array, make_array, + new_empty_array, new_null_array, }; use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer}; use datafusion::arrow::compute::take_arrays; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::{adjust_offsets_for_slice, list_values, take_function_args}; -use datafusion_common::{exec_err, plan_err, Result}; +use datafusion_common::{Result, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaArgument, LambdaParametersProgress, ValueOrLambda, Volatility, @@ -84,12 +84,17 @@ impl HigherOrderUDFImpl for SparkArrayAggregate { fields: &[ValueOrLambda>], ) -> Result { let (list, zero, has_finish) = match fields { - [ValueOrLambda::Value(list), ValueOrLambda::Value(zero), ValueOrLambda::Lambda(_merge)] => { - (list, zero, false) - } - [ValueOrLambda::Value(list), ValueOrLambda::Value(zero), ValueOrLambda::Lambda(_merge), ValueOrLambda::Lambda(_finish)] => { - (list, zero, true) - } + [ + ValueOrLambda::Value(list), + ValueOrLambda::Value(zero), + ValueOrLambda::Lambda(_merge), + ] => (list, zero, false), + [ + ValueOrLambda::Value(list), + ValueOrLambda::Value(zero), + ValueOrLambda::Lambda(_merge), + ValueOrLambda::Lambda(_finish), + ] => (list, zero, true), _ => { return plan_err!( "{} expects value, value, lambda[, lambda] arguments", diff --git a/crates/sail-function/src/scalar/array/spark_array_compact.rs b/crates/sail-function/src/scalar/array/spark_array_compact.rs index f40246b5b..54a490ef6 100644 --- a/crates/sail-function/src/scalar/array/spark_array_compact.rs +++ b/crates/sail-function/src/scalar/array/spark_array_compact.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use datafusion::arrow::array::{ - as_large_list_array, as_list_array, Array, ArrayRef, BooleanArray, GenericListArray, - OffsetSizeTrait, + Array, ArrayRef, BooleanArray, GenericListArray, OffsetSizeTrait, as_large_list_array, + as_list_array, }; use datafusion::arrow::buffer::OffsetBuffer; use datafusion::arrow::compute; use datafusion::arrow::datatypes::DataType; -use datafusion_common::{exec_err, plan_err, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use crate::functions_nested_utils::make_scalar_function; @@ -84,6 +84,7 @@ fn array_compact_inner(args: &[ArrayRef]) -> Result { fn array_compact_generic(list: &GenericListArray) -> Result { let values = list.values(); + let values_nulls = values.logical_nulls(); let offsets = list.offsets(); // Build a boolean filter mask for the flattened values array. @@ -106,7 +107,7 @@ fn array_compact_generic(list: &GenericListArray) -> Resu } else { // Non-null row: keep only non-null values. for j in start..end { - let keep_value = values.is_valid(j); + let keep_value = values_nulls.as_ref().is_none_or(|nulls| nulls.is_valid(j)); keep.push(keep_value); if keep_value { total_kept += 1; diff --git a/crates/sail-function/src/scalar/array/spark_array_exists.rs b/crates/sail-function/src/scalar/array/spark_array_exists.rs index 6ffc0a42b..62add8faa 100644 --- a/crates/sail-function/src/scalar/array/spark_array_exists.rs +++ b/crates/sail-function/src/scalar/array/spark_array_exists.rs @@ -18,13 +18,15 @@ use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer}; use datafusion::arrow::compute::take_arrays; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::{adjust_offsets_for_slice, list_values, list_values_row_number}; -use datafusion_common::{exec_err, plan_err, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, Volatility, }; -use super::lambda_utils::{coerce_single_list_arg, value_lambda_pair}; +use super::lambda_utils::{ + coerce_single_list_arg, short_circuit_boolean_reduce, value_lambda_pair, +}; #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkArrayExists { @@ -86,10 +88,19 @@ impl HigherOrderUDFImpl for SparkArrayExists { let values_param = || Ok(Arc::clone(&values)); let params: [&dyn Fn() -> Result; 1] = [&values_param]; - let predicate_output = lambda.evaluate(¶ms, |arrays| { + let predicate_output = match lambda.evaluate(¶ms, |arrays| { let indices = list_values_row_number(&list_array)?; Ok(take_arrays(arrays, &indices, None)?) - })?; + }) { + Ok(output) => output, + // Evaluating every element at once can raise on an element that Spark + // would never reach; retry in Spark's order before giving up. + Err(_) => { + let result = + short_circuit_boolean_reduce(self.name(), &list_array, &values, lambda, true)?; + return Ok(ColumnarValue::Array(Arc::new(result))); + } + }; let result = match list_array.data_type() { DataType::List(_) => { diff --git a/crates/sail-function/src/scalar/array/spark_array_filter.rs b/crates/sail-function/src/scalar/array/spark_array_filter.rs index 575a213ac..bb822c2da 100644 --- a/crates/sail-function/src/scalar/array/spark_array_filter.rs +++ b/crates/sail-function/src/scalar/array/spark_array_filter.rs @@ -17,21 +17,21 @@ use std::sync::Arc; use datafusion::arrow::array::{ - new_empty_array, Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray, - OffsetBufferBuilder, OffsetSizeTrait, + Array, ArrayRef, AsArray, BooleanArray, LargeListArray, ListArray, OffsetBufferBuilder, + OffsetSizeTrait, new_empty_array, }; use datafusion::arrow::buffer::{OffsetBuffer, ScalarBuffer}; use datafusion::arrow::compute::{filter as arrow_filter, take_arrays}; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::{adjust_offsets_for_slice, list_values_row_number}; -use datafusion_common::{exec_err, plan_err, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, Volatility, }; use crate::scalar::array::lambda_utils::{ - coerce_single_list_arg, extract_list_values, index_array, value_lambda_pair, ListValuesResult, + ListValuesResult, coerce_single_list_arg, extract_list_values, index_array, value_lambda_pair, }; /// The physical lambda evaluation batch is laid out as `[captures..., params...]` @@ -287,7 +287,7 @@ mod tests { use datafusion_common::DFSchema; use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion_physical_expr::create_physical_expr; use super::*; diff --git a/crates/sail-function/src/scalar/array/spark_array_forall.rs b/crates/sail-function/src/scalar/array/spark_array_forall.rs index 715103881..58e252bbb 100644 --- a/crates/sail-function/src/scalar/array/spark_array_forall.rs +++ b/crates/sail-function/src/scalar/array/spark_array_forall.rs @@ -17,13 +17,15 @@ use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer}; use datafusion::arrow::compute::take_arrays; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::{adjust_offsets_for_slice, list_values, list_values_row_number}; -use datafusion_common::{exec_err, plan_err, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, Volatility, }; -use super::lambda_utils::{coerce_single_list_arg, value_lambda_pair}; +use super::lambda_utils::{ + coerce_single_list_arg, short_circuit_boolean_reduce, value_lambda_pair, +}; #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkArrayForall { @@ -85,10 +87,19 @@ impl HigherOrderUDFImpl for SparkArrayForall { let values_param = || Ok(Arc::clone(&values)); let params: [&dyn Fn() -> Result; 1] = [&values_param]; - let predicate_output = lambda.evaluate(¶ms, |arrays| { + let predicate_output = match lambda.evaluate(¶ms, |arrays| { let indices = list_values_row_number(&list_array)?; Ok(take_arrays(arrays, &indices, None)?) - })?; + }) { + Ok(output) => output, + // Evaluating every element at once can raise on an element that Spark + // would never reach; retry in Spark's order before giving up. + Err(_) => { + let result = + short_circuit_boolean_reduce(self.name(), &list_array, &values, lambda, false)?; + return Ok(ColumnarValue::Array(Arc::new(result))); + } + }; let result = match list_array.data_type() { DataType::List(_) => { diff --git a/crates/sail-function/src/scalar/array/spark_array_min_max.rs b/crates/sail-function/src/scalar/array/spark_array_min_max.rs index 750a3d70d..39ee59379 100644 --- a/crates/sail-function/src/scalar/array/spark_array_min_max.rs +++ b/crates/sail-function/src/scalar/array/spark_array_min_max.rs @@ -1,9 +1,9 @@ use datafusion::arrow::array::{ - as_large_list_array, as_list_array, Array, ArrayRef, GenericListArray, OffsetSizeTrait, + Array, ArrayRef, GenericListArray, OffsetSizeTrait, as_large_list_array, as_list_array, }; use datafusion::arrow::datatypes::DataType; use datafusion::functions_aggregate::min_max; -use datafusion_common::{exec_err, plan_err, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ Accumulator, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; diff --git a/crates/sail-function/src/scalar/array/spark_array_sort.rs b/crates/sail-function/src/scalar/array/spark_array_sort.rs index 68b2645cf..86e50aae0 100644 --- a/crates/sail-function/src/scalar/array/spark_array_sort.rs +++ b/crates/sail-function/src/scalar/array/spark_array_sort.rs @@ -24,14 +24,14 @@ use datafusion::arrow::array::{ use datafusion::arrow::compute::{take, take_arrays}; use datafusion::arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef}; use datafusion_common::utils::adjust_offsets_for_slice; -use datafusion_common::{exec_datafusion_err, exec_err, plan_err, DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, exec_datafusion_err, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, Volatility, }; use crate::scalar::array::lambda_utils::{ - coerce_single_list_arg, extract_list_values, value_lambda_pair, ListValuesResult, + ListValuesResult, coerce_single_list_arg, extract_list_values, value_lambda_pair, }; /// The physical lambda evaluation batch is laid out as `[captures..., params...]` @@ -220,8 +220,7 @@ impl HigherOrderUDFImpl for SparkArraySort { }; let mut perm: Vec = Vec::with_capacity(list_values.len()); - for (row, w) in offsets.windows(2).enumerate() { - let (start, end) = (w[0], w[1]); + for (row, &[start, end]) in offsets.array_windows::<2>().enumerate() { let n = end - start; if n == 0 { continue; @@ -289,7 +288,7 @@ mod tests { use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion_expr::{col, lambda, lit, Case, Expr, HigherOrderUDF}; + use datafusion_expr::{Case, Expr, HigherOrderUDF, col, lambda, lit}; use datafusion_physical_expr::create_physical_expr; use super::*; diff --git a/crates/sail-function/src/scalar/array/spark_array_transform.rs b/crates/sail-function/src/scalar/array/spark_array_transform.rs index 5ae34dc90..c1ba66c33 100644 --- a/crates/sail-function/src/scalar/array/spark_array_transform.rs +++ b/crates/sail-function/src/scalar/array/spark_array_transform.rs @@ -20,14 +20,14 @@ use datafusion::arrow::array::{Array, ArrayRef, AsArray, LargeListArray, ListArr use datafusion::arrow::compute::take_arrays; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::{adjust_offsets_for_slice, list_values_row_number}; -use datafusion_common::{exec_err, plan_err, Result}; +use datafusion_common::{Result, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, Volatility, }; use crate::scalar::array::lambda_utils::{ - coerce_single_list_arg, extract_list_values, index_array, value_lambda_pair, ListValuesResult, + ListValuesResult, coerce_single_list_arg, extract_list_values, index_array, value_lambda_pair, }; /// Like [`super::spark_array_filter::SparkArrayFilter`], the physical lambda @@ -215,7 +215,7 @@ mod tests { use datafusion_common::DFSchema; use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::expr::{HigherOrderFunction, LambdaVariable}; - use datafusion_expr::{col, lambda, lit, Expr, HigherOrderUDF}; + use datafusion_expr::{Expr, HigherOrderUDF, col, lambda, lit}; use datafusion_physical_expr::create_physical_expr; use super::*; diff --git a/crates/sail-function/src/scalar/array/spark_sequence.rs b/crates/sail-function/src/scalar/array/spark_sequence.rs index 9c43e1df1..b2e3056b7 100644 --- a/crates/sail-function/src/scalar/array/spark_sequence.rs +++ b/crates/sail-function/src/scalar/array/spark_sequence.rs @@ -4,16 +4,16 @@ use std::sync::Arc; use datafusion::arrow::array::timezone::Tz; use datafusion::arrow::array::{ - Array, ArrayRef, AsArray, Date32Builder, Int16Array, Int32Array, Int64Array, Int8Array, + Array, ArrayRef, AsArray, Date32Builder, Int8Array, Int16Array, Int32Array, Int64Array, ListArray, ListBuilder, NullBufferBuilder, TimestampMicrosecondBuilder, }; use datafusion::arrow::buffer::OffsetBuffer; use datafusion::arrow::datatypes::{ - ArrowNativeType, DataType, Date32Type, Field, Int16Type, Int32Type, Int64Type, Int8Type, + ArrowNativeType, DataType, Date32Type, Field, Int8Type, Int16Type, Int32Type, Int64Type, IntervalMonthDayNanoType, IntervalUnit, TimeUnit, TimestampMicrosecondType, }; use datafusion::arrow::temporal_conversions::as_datetime_with_timezone; -use datafusion_common::{exec_datafusion_err, exec_err, internal_err, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err, internal_err}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use crate::functions_nested_utils::make_scalar_function; @@ -192,10 +192,12 @@ fn gen_sequence_timestamp(args: &[ArrayRef]) -> Result { ), )?; let mut current = start; - let mut current_dt = as_datetime_with_timezone::(current, start_tz) - .ok_or(exec_datafusion_err!( - "Spark `sequence` function cannot generate timestamp for start: {current}: {start_tz:?}" - ))?; + let mut current_dt = as_datetime_with_timezone::( + current, start_tz, + ) + .ok_or(exec_datafusion_err!( + "Spark `sequence` function cannot generate timestamp for start: {current}: {start_tz:?}" + ))?; let values = from_fn(|| { if (negative && current_dt < stop_dt) || (!negative && current_dt > stop_dt) { @@ -347,7 +349,7 @@ fn gen_sequence_date(args: &[ArrayRef]) -> Result { } macro_rules! impl_sequence_for_type { - ($name:ident, $native_type:ty, $array_type:ty, $array_builder:ty, $data_type:expr, $zero:expr, $one:expr, $max:expr) => { + ($name:ident, $native_type:ty, $array_type:ty, $array_builder:ty, $data_type:expr_2021, $zero:expr_2021, $one:expr_2021, $max:expr_2021) => { fn $name(args: &[ArrayRef]) -> Result { let (start_array, stop_array, step_array) = match args.len() { 1 => (None, args[0].as_primitive::<$array_type>(), None), diff --git a/crates/sail-function/src/scalar/array_struct_field.rs b/crates/sail-function/src/scalar/array_struct_field.rs index 3bb2d9f23..c7ea11394 100644 --- a/crates/sail-function/src/scalar/array_struct_field.rs +++ b/crates/sail-function/src/scalar/array_struct_field.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use datafusion::arrow::array::{ - make_array, Array, ArrayRef, FixedSizeListArray, GenericListArray, OffsetSizeTrait, StructArray, + Array, ArrayRef, FixedSizeListArray, GenericListArray, OffsetSizeTrait, StructArray, make_array, }; use datafusion::arrow::buffer::NullBuffer; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::{ - exec_datafusion_err, exec_err, internal_err, plan_datafusion_err, Result, ScalarValue, + Result, ScalarValue, exec_datafusion_err, exec_err, internal_err, plan_datafusion_err, }; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -49,6 +49,16 @@ impl ArrayStructField { } } + fn list_item_field(field: &FieldRef, nullable: bool) -> FieldRef { + Arc::new( + field + .as_ref() + .clone() + .with_name(Field::LIST_FIELD_DEFAULT_NAME) + .with_nullable(nullable), + ) + } + fn return_type_for_list(&self, data_type: &DataType, field_name: &str) -> Result { let list_field = Self::list_field(data_type)?; let DataType::Struct(fields) = list_field.data_type() else { @@ -61,7 +71,7 @@ impl ArrayStructField { .find(field_name) .ok_or_else(|| exec_datafusion_err!("missing field: {field_name}"))?; let nullable = list_field.is_nullable() || child.is_nullable(); - let field = Arc::new(Field::new_list_field(child.data_type().clone(), nullable)); + let field = Self::list_item_field(child, nullable); match data_type { DataType::List(_) => Ok(DataType::List(field)), DataType::LargeList(_) => Ok(DataType::LargeList(field)), @@ -70,7 +80,7 @@ impl ArrayStructField { } } - fn struct_values(values: &ArrayRef, field_name: &str) -> Result<(ArrayRef, bool)> { + fn struct_values(values: &ArrayRef, field_name: &str) -> Result<(ArrayRef, FieldRef)> { let Some(struct_array) = values.as_any().downcast_ref::() else { return exec_err!( "array_struct_field requires list values to be structs, got {}", @@ -84,7 +94,7 @@ impl ArrayStructField { let values = struct_array.column(index); let nulls = NullBuffer::union(struct_array.nulls(), values.nulls()); let data = values.to_data().into_builder().nulls(nulls).build()?; - Ok((make_array(data), field.is_nullable())) + Ok((make_array(data), Arc::clone(field))) } fn project_list(array: &GenericListArray, field_name: &str) -> Result @@ -92,12 +102,9 @@ impl ArrayStructField { O: OffsetSizeTrait, { let list_field = Self::list_field(array.data_type())?; - let (values, child_nullable) = Self::struct_values(array.values(), field_name)?; + let (values, child) = Self::struct_values(array.values(), field_name)?; Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new_list_field( - values.data_type().clone(), - list_field.is_nullable() || child_nullable, - )), + Self::list_item_field(&child, list_field.is_nullable() || child.is_nullable()), array.offsets().clone(), values, array.nulls().cloned(), @@ -110,12 +117,9 @@ impl ArrayStructField { field_name: &str, ) -> Result { let list_field = Self::list_field(array.data_type())?; - let (values, child_nullable) = Self::struct_values(array.values(), field_name)?; + let (values, child) = Self::struct_values(array.values(), field_name)?; Ok(Arc::new(FixedSizeListArray::try_new( - Arc::new(Field::new_list_field( - values.data_type().clone(), - list_field.is_nullable() || child_nullable, - )), + Self::list_item_field(&child, list_field.is_nullable() || child.is_nullable()), size, values, array.nulls().cloned(), @@ -184,7 +188,7 @@ impl ScalarUDFImpl for ArrayStructField { return exec_err!( "array_struct_field requires a list, got {}", array.data_type() - ) + ); } }; Ok(ColumnarValue::Array(out)) diff --git a/crates/sail-function/src/scalar/collection/spark_concat.rs b/crates/sail-function/src/scalar/collection/spark_concat.rs index 861a75172..ec6ea5477 100644 --- a/crates/sail-function/src/scalar/collection/spark_concat.rs +++ b/crates/sail-function/src/scalar/collection/spark_concat.rs @@ -5,16 +5,17 @@ use datafusion::arrow::array::{BooleanArray, StringArray}; use datafusion::arrow::compute::cast; use datafusion::arrow::compute::kernels::boolean::{is_null, or}; use datafusion::arrow::compute::kernels::nullif::nullif; -use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion::functions::string::concat::ConcatFunc; use datafusion_common::utils::list_ndims; -use datafusion_common::{plan_err, ExprSchema, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue, internal_err, plan_err}; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ - ColumnarValue, Expr, ExprSchemable, ScalarFunctionArgs, ScalarUDFImpl, Signature, + ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; use datafusion_functions_nested::concat::ArrayConcat; +use sail_common_datafusion::utils::items::ItemTaker; #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkConcat { @@ -47,86 +48,75 @@ impl ScalarUDFImpl for SparkConcat { &self.signature } - /// [Credit]: - fn return_type(&self, arg_types: &[DataType]) -> Result { - if arg_types.is_empty() { - return Ok(DataType::Utf8); - } - if arg_types - .iter() - .any(|arg_type| matches!(arg_type, DataType::List(_))) - { - let mut expr_type: Option = None; - let mut max_dims = 0; - for arg_type in arg_types { - match arg_type { - DataType::List(field) => { - if !field.data_type().equals_datatype(&DataType::Null) { - let dims = list_ndims(arg_type); - expr_type = Some(match max_dims.cmp(&dims) { - Ordering::Greater => expr_type.unwrap_or_else(|| arg_type.clone()), - Ordering::Equal => { - if let Some(expr_type) = expr_type { - merge_list_types(&expr_type, arg_type).ok_or_else(|| { - datafusion_common::plan_datafusion_err!( - "It is not possible to concatenate arrays of different types. Expected: {expr_type}, got: {arg_type}" - ) - })? - } else { - arg_type.clone() - } - } - Ordering::Less => { - max_dims = dims; - arg_type.clone() - } - }); - } - } - _ => { - return plan_err!( - "The array_concat function can only accept list as the args." - ) - } - } - } - // All arrays had Null element type (e.g. array() + array()) — keep as List(Null) - Ok(expr_type.unwrap_or_else(|| arg_types[0].clone())) - } else if arg_types + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!( + "spark_concat: `return_type` should not be called; `return_field_from_args` is used instead" + ) + } + + /// `concat` propagates NULL: the result is null iff any argument is null + /// (mirrors Spark `Concat.nullable`). Set here rather than in the deprecated + /// `is_nullable`, which DataFusion no longer consults to derive the field. + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let data_types = args + .arg_fields .iter() - .all(|arg_type| matches!(arg_type, DataType::Binary)) - { - Ok(DataType::Binary) - } else if arg_types + .map(|field| field.data_type().clone()) + .collect::>(); + let return_type = concat_return_type(&data_types)?; + let nullable = args.arg_fields.iter().any(|field| field.is_nullable()); + Ok(Arc::new(Field::new(self.name(), return_type, nullable))) + } + + /// Plan-time simplifications that mirror Spark's `concat` semantics. + /// + /// concat — simplify summary: + /// ```text + /// concat(concat(a, b), c) -> concat(a, b, c) (R1: CombineConcats — flatten nested) + /// concat(a, NULL, b) -> NULL (R2: null propagation — any null arg) + /// concat(x) -> x (R3: single-arg identity: string/array/binary) + /// ``` + /// + /// * **R1** — flatten nested `concat` calls first, so deeply nested calls + /// collapse in one bottom-up pass and a NULL bubbled up from an inner call + /// is caught by R2. Only flattens when the return type is preserved. + /// * **R2** — `concat` propagates NULL: a literal NULL argument folds the whole + /// call to a typed NULL at planning time, eliminating the other argument + /// subtrees (DCE) and enabling column pruning. Falls through on a type-invalid + /// combo (e.g. array + untyped NULL) so the normal path still raises the error. + /// * **R3** — single-argument identity for string/array/binary inputs; other + /// types fall through so the invoke path can apply Spark coercions. + fn simplify(&self, args: Vec, info: &SimplifyContext) -> Result { + // R1: flatten nested concat (Spark's CombineConcats). + let args = flatten_nested_concats(args, info)?; + + // R2: null propagation. Only fold when the arg types form a valid concat — + // otherwise fall through so the normal path still raises the type error + // (e.g. array + untyped NULL). + if args .iter() - .all(|arg_type| matches!(arg_type, DataType::Binary | DataType::LargeBinary)) + .any(|arg| matches!(arg, Expr::Literal(scalar, _) if scalar.is_null())) { - Ok(DataType::LargeBinary) - } else { - Ok(arg_types + let arg_types = args .iter() - .find(|&arg_type| matches!(arg_type, &DataType::Utf8View)) - .or_else(|| { - arg_types - .iter() - .find(|&arg_type| matches!(arg_type, &DataType::LargeUtf8)) - }) - .unwrap_or(&DataType::Utf8) - .clone()) + .map(|arg| info.get_data_type(arg)) + .collect::>>()?; + if let Ok(null) = concat_return_type(&arg_types) + .and_then(|data_type| ScalarValue::try_from(&data_type)) + { + return Ok(ExprSimplifyResult::Simplified(Expr::Literal(null, None))); + } } - } - fn simplify(&self, args: Vec, info: &SimplifyContext) -> Result { if args.len() != 1 { return Ok(ExprSimplifyResult::Original(args)); } - let mut args = args; - let arg = args.remove(0); - // `FixedSizeList` and `LargeList` are intentionally omitted: `return_type` - // does not recognize them as array inputs (it falls through to `Utf8`), - // so the invoke path coerces them to strings. Returning the argument - // unchanged here would break the type contract advertised by - // `return_type`. Numeric, timestamp, and other non-string scalars are + let arg = args.one()?; + // `FixedSizeList` and `LargeList` are intentionally omitted: + // `concat_return_type` does not recognize them as array inputs (it falls + // through to `Utf8`), so the invoke path coerces them to strings. Returning + // the argument unchanged here would break the type contract advertised by + // `concat_return_type`. Numeric, timestamp, and other non-string scalars are // also excluded so the invoke path can apply Spark-specific coercions // (e.g. timestamp formatting via `spark_format_timestamp_str`). if matches!( @@ -145,29 +135,23 @@ impl ScalarUDFImpl for SparkConcat { } } - fn is_nullable(&self, args: &[Expr], schema: &dyn ExprSchema) -> bool { - if args.is_empty() { - true - } else { - args.iter().any(|arg| arg.nullable(schema).unwrap_or(true)) - } - } - fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() { return Ok(ColumnarValue::Scalar(ScalarValue::Utf8( Some(String::new()), ))); } + // `return_type` is cloned because `args.return_field` is moved into the + // delegate call below, but the null-typed early-return still needs it. let return_type = args.return_field.data_type().clone(); let mut null_mask = None; - for arg in args.args.clone() { + for arg in &args.args { match arg { ColumnarValue::Scalar(s) if s.is_null() => { return Ok(ColumnarValue::Scalar(ScalarValue::try_from(&return_type)?)); } ColumnarValue::Array(a) => { - let mask = is_null(&a)?; + let mask = is_null(a)?; null_mask = match null_mask { Some(existing) => Some(or(&existing, &mask)?), None => Some(mask), @@ -270,21 +254,17 @@ fn cast_columnar_values( /// "2024-01-15 12:00:00+08:00"). Spark uses a space separator and no timezone /// suffix (e.g. "2024-01-15 12:00:00"). fn spark_format_timestamp_str(s: &str) -> String { - // Replace ISO 8601 'T' separator with a space - let s = s.replace('T', " "); - // Strip timezone suffix (Z, +HH:MM, -HH:MM) that starts after the seconds - // part. "YYYY-MM-DD HH:MM:SS" is 19 chars, so search from position 19 to - // avoid matching the '-' in the date portion. - let cutoff = s + // Strip the timezone suffix (Z, +HH:MM, -HH:MM) that starts after the seconds + // part. "YYYY-MM-DD HH:MM:SS" is 19 chars, so search from position 19 to avoid + // matching the '-' in the date portion. The cutoff is the same on `s` as on the + // 'T'-replaced string (the 'T' is at index 10, before 19), so we slice first and + // replace once — a single allocation instead of replace-then-slice. + let end = s .char_indices() .skip_while(|(i, _)| *i < 19) .find(|(_, c)| matches!(c, 'Z' | '+' | '-')) - .map(|(i, _)| i); - if let Some(idx) = cutoff { - s[..idx].to_string() - } else { - s - } + .map_or(s.len(), |(i, _)| i); + s[..end].replace('T', " ") } fn spark_format_timestamp_scalar(scalar: ScalarValue) -> Result { @@ -345,6 +325,120 @@ fn cast_list_columnar_values( .collect() } +/// If `expr` is a nested `spark_concat` call, return its arguments. +fn nested_concat_args(expr: &Expr) -> Option<&[Expr]> { + match expr { + Expr::ScalarFunction(inner) + if inner.func.inner().downcast_ref::().is_some() => + { + Some(&inner.args) + } + _ => None, + } +} + +/// Spark's `CombineConcats`: flatten nested `spark_concat` children, +/// `concat(concat(a, b), c)` -> `concat(a, b, c)`, reducing nesting to a single +/// kernel pass. Only flatten when it keeps the same return type — this guards the +/// array element-type / dimension merge in `return_type`; otherwise the arguments +/// are returned unchanged. +fn flatten_nested_concats(args: Vec, info: &SimplifyContext) -> Result> { + if !args.iter().any(|arg| nested_concat_args(arg).is_some()) { + return Ok(args); + } + let mut flattened = Vec::with_capacity(args.len()); + for arg in &args { + match nested_concat_args(arg) { + Some(inner) => flattened.extend(inner.iter().cloned()), + None => flattened.push(arg.clone()), + } + } + let before = args + .iter() + .map(|arg| info.get_data_type(arg)) + .collect::>>()?; + let after = flattened + .iter() + .map(|arg| info.get_data_type(arg)) + .collect::>>()?; + match (concat_return_type(&before), concat_return_type(&after)) { + (Ok(before_type), Ok(after_type)) if before_type == after_type => Ok(flattened), + _ => Ok(args), + } +} + +/// Spark `concat` output type: array concat (list dim/element merge), all-binary, +/// or string (widest of Utf8View / LargeUtf8 / Utf8). +/// +/// [Credit]: +fn concat_return_type(arg_types: &[DataType]) -> Result { + if arg_types.is_empty() { + return Ok(DataType::Utf8); + } + if arg_types + .iter() + .any(|arg_type| matches!(arg_type, DataType::List(_))) + { + let mut expr_type: Option = None; + let mut max_dims = 0; + for arg_type in arg_types { + match arg_type { + DataType::List(field) => { + if !field.data_type().equals_datatype(&DataType::Null) { + let dims = list_ndims(arg_type); + expr_type = Some(match max_dims.cmp(&dims) { + Ordering::Greater => expr_type.unwrap_or_else(|| arg_type.clone()), + Ordering::Equal => { + if let Some(expr_type) = expr_type { + merge_list_types(&expr_type, arg_type).ok_or_else(|| { + datafusion_common::plan_datafusion_err!( + "It is not possible to concatenate arrays of different types. Expected: {expr_type}, got: {arg_type}" + ) + })? + } else { + arg_type.clone() + } + } + Ordering::Less => { + max_dims = dims; + arg_type.clone() + } + }); + } + } + _ => { + return plan_err!( + "The array_concat function can only accept list as the args." + ); + } + } + } + // All arrays had Null element type (e.g. array() + array()) — keep as List(Null) + Ok(expr_type.unwrap_or_else(|| arg_types[0].clone())) + } else if arg_types + .iter() + .all(|arg_type| matches!(arg_type, DataType::Binary)) + { + Ok(DataType::Binary) + } else if arg_types + .iter() + .all(|arg_type| matches!(arg_type, DataType::Binary | DataType::LargeBinary)) + { + Ok(DataType::LargeBinary) + } else { + Ok(arg_types + .iter() + .find(|&arg_type| matches!(arg_type, &DataType::Utf8View)) + .or_else(|| { + arg_types + .iter() + .find(|&arg_type| matches!(arg_type, &DataType::LargeUtf8)) + }) + .unwrap_or(&DataType::Utf8) + .clone()) + } +} + fn merge_list_types(left: &DataType, right: &DataType) -> Option { match (left, right) { (DataType::List(left), DataType::List(right)) => { diff --git a/crates/sail-function/src/scalar/collection/spark_reverse.rs b/crates/sail-function/src/scalar/collection/spark_reverse.rs index 41c0f6d53..9d8e51477 100644 --- a/crates/sail-function/src/scalar/collection/spark_reverse.rs +++ b/crates/sail-function/src/scalar/collection/spark_reverse.rs @@ -1,6 +1,6 @@ use datafusion::arrow::datatypes::DataType; use datafusion::functions::unicode::reverse::ReverseFunc; -use datafusion_common::{exec_err, Result}; +use datafusion_common::{Result, exec_err}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use datafusion_functions_nested::reverse::array_reverse_inner; diff --git a/crates/sail-function/src/scalar/csv/mod.rs b/crates/sail-function/src/scalar/csv/mod.rs index 08f5bc310..2695b8de9 100644 --- a/crates/sail-function/src/scalar/csv/mod.rs +++ b/crates/sail-function/src/scalar/csv/mod.rs @@ -1,3 +1,4 @@ +mod options; mod schema_of_csv; pub mod spark_from_csv; pub mod spark_to_csv; diff --git a/crates/sail-function/src/scalar/csv/options.rs b/crates/sail-function/src/scalar/csv/options.rs new file mode 100644 index 000000000..fc0773b6f --- /dev/null +++ b/crates/sail-function/src/scalar/csv/options.rs @@ -0,0 +1,330 @@ +use std::ops::Range; + +use datafusion::arrow::array::{Array, MapArray, StringArray}; +use datafusion::error::Result; +use datafusion_common::exec_err; +use sail_common::spec::{SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME}; + +/// Which CSV expression function is reading the options. +/// +/// Spark validates almost every option identically in all three, but not quite: only the writer +/// rejects an empty `lineSep`, and only `from_csv` rejects the `DROPMALFORMED` parse mode. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum CsvFunction { + From, + To, + SchemaOf, +} + +impl CsvFunction { + fn name(self) -> &'static str { + match self { + Self::From => "from_csv", + Self::To => "to_csv", + Self::SchemaOf => "schema_of_csv", + } + } +} + +/// The boolean options that Spark's `CSVOptions` parses with `getBool`, which reports +/// `

^|[^%])` prefix in each regex prevents re-matching a
-    /// letter that is immediately preceded by `%`. This protects the
-    /// canonical chrono directives (`%Y`, `%m`, `%H`, ...) from being
-    /// re-substituted as Spark tokens. It does NOT protect letters produced
-    /// via the short `%-X` forms (`%-m`, `%-d`, `%-H`, ...), where the
-    /// meaningful letter sits after a `-` and remains eligible for later
-    /// matching.
-    ///
-    /// IMPORTANT: pattern order is load-bearing. Longer patterns within each
-    /// family (e.g., `SSSSSSSSS` before `S`, `MMMM` before `M`) MUST come
-    /// first because `replace_all` is applied sequentially per regex.
-    /// Do not alphabetize.
-    static ref CHRONO_REPLACEMENTS: Vec<(Regex, String)> = {
-        let patterns: &[(&str, &str)] = &[
-            // Fractional seconds patterns (from nanoseconds to deciseconds)
-            ("SSSSSSSSS", "%.9f"), // Nanoseconds
-            ("SSSSSSSS", "%.8f"),
-            ("SSSSSSS", "%.7f"),
-            ("SSSSSS", "%.6f"), // Microseconds
-            ("SSSSS", "%.5f"),
-            ("SSSS", "%.4f"),
-            ("SSS", "%.3f"), // Milliseconds
-            ("SS", "%.2f"),  // Centiseconds
-            ("S", "%.1f"),   // Deciseconds
-            // Year patterns
-            ("yyyy", "%Y"),
-            ("yyy", "%Y"),
-            ("yy", "%y"),
-            ("y", "%Y"),
-            // Day-of-year pattern
-            ("D", "%j"),
-            // Month patterns
-            ("MMMM", "%B"),
-            ("MMM", "%b"),
-            ("MM", "%m"),
-            ("M", "%-m"),
-            ("LLLL", "%B"),
-            ("LLL", "%b"),
-            ("LL", "%m"),
-            ("L", "%-m"),
-            // Day-of-month patterns
-            ("dd", "%d"),
-            ("d", "%-d"),
-            // Weekday patterns
-            ("EEEE", "%A"),
-            ("EEE", "%a"),
-            ("E", "%a"),
-            // Hour patterns
-            ("hh", "%I"), // 12-hour clock (01–12)
-            ("h", "%-I"), // 12-hour clock (1–12)
-            ("HH", "%H"), // 24-hour clock (00–23)
-            ("H", "%-H"), // 24-hour clock (0–23)
-            ("KK", "%I"), // 12-hour clock (01–12), but Spark's 'K' is 0–11
-            ("K", "%l"),  // 12-hour clock (1–12), space-padded
-            // Minute patterns
-            ("mm", "%M"),
-            ("m", "%-M"),
-            // Second patterns
-            ("ss", "%S"),
-            ("s", "%-S"),
-            // AM/PM
-            ("a", "%p"),
-            // Timezone patterns
-            ("XXXXX", "%::z"), // ±HH:MM:SS
-            ("XXXX", "%z"),    // ±HHMM
-            ("XXX", "%:z"),    // ±HH:MM
-            ("XX", "%z"),      // ±HHMM
-            ("X", "%z"),       // ±HHMM
-            ("xxxxx", "%::z"), // ±HH:MM:SS
-            ("xxxx", "%z"),    // ±HHMM
-            ("xxx", "%:z"),    // ±HH:MM
-            ("xx", "%z"),      // ±HHMM
-            ("x", "%z"),       // ±HHMM
-            ("ZZZZZ", "%::z"), // ±HH:MM:SS
-            ("ZZZZ", "%:z"),   // ±HH:MM
-            ("ZZZ", "%z"),     // ±HHMM
-            ("ZZ", "%z"),      // ±HHMM
-            ("Z", "%z"),       // ±HHMM
-            ("zzzz", "%Z"),
-            ("zzz", "%Z"),
-            ("zz", "%Z"),
-            ("z", "%Z"),
-            ("OOOO", "%Z"),
-            ("OO", "%Z"),
-            ("VV", "%Z"),
-        ];
-        patterns
-            .iter()
-            .map(|(pattern, replacement)| {
-                let pattern = format!(r"(?P
^|[^%]){}", regex::escape(pattern));
-                #[expect(clippy::expect_used)]
-                let re = Regex::new(&pattern).expect("valid datetime pattern");
-                // Precompute the FULL replacement string including the
-                // `${pre}` capture group reference to avoid per-call allocation.
-                let replacement = format!("${{pre}}{replacement}");
-                (re, replacement)
-            })
-            .collect()
-    };
-}
-
-pub fn spark_datetime_format_to_chrono_strftime(format: &str) -> Result {
-    // TODO: This doesn't cover everything.
-    //  https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-    //  https://docs.rs/chrono/latest/chrono/format/strftime/index.html#specifiers
-
-    let mut result = format.to_string();
-    for (re, replacement) in CHRONO_REPLACEMENTS.iter() {
-        // `replace_all` returns `Cow::Borrowed` when nothing matched; in that
-        // case the input is unchanged and there is no need to allocate a new
-        // `String`. Only reassign `result` when an actual substitution
-        // happened (`Cow::Owned`).
-        if let std::borrow::Cow::Owned(x) = re.replace_all(&result, replacement.as_str()) {
-            result = x;
-        }
-    }
-
-    // Fix double-dot issue: chrono's %.Nf already includes a leading dot,
-    // so when the Spark format has a literal '.' before S-patterns (e.g., "ss.SSS"),
-    // the result would have ".%.Nf" which produces "..NNN". Remove the extra dot.
-    result = result.replace(".%.", "%.");
-
-    Ok(result)
-}
 
 // Shared array conversion helpers for make_timestamp functions
 
@@ -214,43 +62,3 @@ pub(crate) fn to_int32_array(
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_convert_spark_datetime_format_to_chrono() -> Result<()> {
-        assert_eq!(
-            spark_datetime_format_to_chrono_strftime("yyyy-MM-dd HH:mm:ss")?,
-            "%Y-%m-%d %H:%M:%S"
-        );
-        assert_eq!(
-            spark_datetime_format_to_chrono_strftime("ss.SSS")?,
-            "%S%.3f"
-        );
-        assert_eq!(spark_datetime_format_to_chrono_strftime("MM mm")?, "%m %M");
-        assert_eq!(
-            spark_datetime_format_to_chrono_strftime("EEEE EEE E")?,
-            "%A %a %a"
-        );
-        assert_eq!(spark_datetime_format_to_chrono_strftime("SSSSS")?, "%.5f");
-        // This is a bug. Fixing the bug would require updating this test case.
-        assert_eq!(
-            spark_datetime_format_to_chrono_strftime("MMMM-MMM-MM-M")?,
-            "%B-%b-%m-%-%-M"
-        );
-        assert_eq!(spark_datetime_format_to_chrono_strftime("")?, "");
-        assert_eq!(spark_datetime_format_to_chrono_strftime(" ")?, " ");
-        // This is a bug. `'T'` is literal string.
-        assert_eq!(
-            spark_datetime_format_to_chrono_strftime("yyyy-MM-dd'T'HH:mm:ss.SSSZ")?,
-            "%Y-%m-%d'T'%H:%M:%S%.3f%z"
-        );
-        assert_eq!(
-            spark_datetime_format_to_chrono_strftime("yyyy年MM月dd日")?,
-            "%Y年%m月%d日"
-        );
-        Ok(())
-    }
-}
diff --git a/crates/sail-function/src/scalar/drop_struct_field.rs b/crates/sail-function/src/scalar/drop_struct_field.rs
index 19ecf6fce..25573a7c7 100644
--- a/crates/sail-function/src/scalar/drop_struct_field.rs
+++ b/crates/sail-function/src/scalar/drop_struct_field.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{Array, ArrayRef, StructArray};
 use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion_common::cast::as_struct_array;
-use datafusion_common::{exec_err, plan_err, Result};
+use datafusion_common::{Result, exec_err, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -33,11 +33,9 @@ impl DropStructField {
 
                 let current_field = &field_names[0];
                 let mut new_fields = Vec::new();
-                let mut field_found = false;
 
                 for field in fields.iter() {
                     if field.name() == current_field {
-                        field_found = true;
                         if field_names.len() == 1 {
                             continue;
                         } else {
@@ -54,8 +52,11 @@ impl DropStructField {
                     }
                 }
 
-                if !field_found {
-                    plan_err!("Field `{current_field}` not found")
+                // Spark's `dropFields` silently ignores field names that do not
+                // exist (the struct is returned unchanged), but raises when every
+                // field would be dropped.
+                if new_fields.is_empty() {
+                    plan_err!("[CANNOT_DROP_ALL_FIELDS] Cannot drop all fields in struct")
                 } else {
                     Ok(DataType::Struct(new_fields.into()))
                 }
diff --git a/crates/sail-function/src/scalar/geo/st_asbinary.rs b/crates/sail-function/src/scalar/geo/st_asbinary.rs
index 1c5dbd057..e2769b22f 100644
--- a/crates/sail-function/src/scalar/geo/st_asbinary.rs
+++ b/crates/sail-function/src/scalar/geo/st_asbinary.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use datafusion::arrow::datatypes::{DataType, Field};
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
diff --git a/crates/sail-function/src/scalar/geo/st_geogfromwkb.rs b/crates/sail-function/src/scalar/geo/st_geogfromwkb.rs
index e088e6996..66832d210 100644
--- a/crates/sail-function/src/scalar/geo/st_geogfromwkb.rs
+++ b/crates/sail-function/src/scalar/geo/st_geogfromwkb.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion::common::cast::as_binary_array;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
@@ -85,10 +85,10 @@ impl ScalarUDFImpl for StGeogFromWKB {
             ColumnarValue::Array(array) => {
                 let binary_array = as_binary_array(array)?;
                 for (i, opt) in binary_array.iter().enumerate() {
-                    if let Some(b) = opt {
-                        if let Err(e) = validate_geography(b) {
-                            return exec_err!("Invalid WKB at index {}: {}", i, e);
-                        }
+                    if let Some(b) = opt
+                        && let Err(e) = validate_geography(b)
+                    {
+                        return exec_err!("Invalid WKB at index {}: {}", i, e);
                     }
                 }
                 Ok(ColumnarValue::Array(array.clone()))
diff --git a/crates/sail-function/src/scalar/geo/st_geomfromwkb.rs b/crates/sail-function/src/scalar/geo/st_geomfromwkb.rs
index 7c91f66d3..262d6bc4b 100644
--- a/crates/sail-function/src/scalar/geo/st_geomfromwkb.rs
+++ b/crates/sail-function/src/scalar/geo/st_geomfromwkb.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion::common::cast::as_binary_array;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
@@ -85,10 +85,10 @@ impl ScalarUDFImpl for StGeomFromWKB {
             ColumnarValue::Array(array) => {
                 let binary_array = as_binary_array(array)?;
                 for (i, opt) in binary_array.iter().enumerate() {
-                    if let Some(b) = opt {
-                        if let Err(e) = validate_geometry(b) {
-                            return exec_err!("Invalid WKB at index {}: {}", i, e);
-                        }
+                    if let Some(b) = opt
+                        && let Err(e) = validate_geometry(b)
+                    {
+                        return exec_err!("Invalid WKB at index {}: {}", i, e);
                     }
                 }
                 Ok(ColumnarValue::Array(array.clone()))
diff --git a/crates/sail-function/src/scalar/geo/wkb_reader.rs b/crates/sail-function/src/scalar/geo/wkb_reader.rs
index 7accf36f9..0b2884b05 100644
--- a/crates/sail-function/src/scalar/geo/wkb_reader.rs
+++ b/crates/sail-function/src/scalar/geo/wkb_reader.rs
@@ -27,7 +27,9 @@ pub enum WkbError {
     InvalidType(i32),
     #[error("Unsupported dimension: {0}")]
     UnsupportedDimension(i32),
-    #[error("Unexpected end of WKB buffer: expected {expected} bytes at position {position}, but only {remaining} remaining")]
+    #[error(
+        "Unexpected end of WKB buffer: expected {expected} bytes at position {position}, but only {remaining} remaining"
+    )]
     UnexpectedEndOfBuffer {
         expected: usize,
         remaining: usize,
@@ -37,7 +39,9 @@ pub enum WkbError {
     NonFiniteCoordinate { value: f64, position: usize },
     #[error("Invalid coordinate value at position {position}: {message}")]
     InvalidCoordinateValue { message: String, position: usize },
-    #[error("Too few points in linestring at position {position}: expected at least {expected_min}, got {actual}")]
+    #[error(
+        "Too few points in linestring at position {position}: expected at least {expected_min}, got {actual}"
+    )]
     TooFewPointsInLineString {
         expected_min: i32,
         actual: i32,
@@ -45,7 +49,9 @@ pub enum WkbError {
     },
     #[error("Ring is not closed at position {position}")]
     RingNotClosed { position: usize },
-    #[error("Too few points in ring at position {position}: expected at least {expected_min}, got {actual}")]
+    #[error(
+        "Too few points in ring at position {position}: expected at least {expected_min}, got {actual}"
+    )]
     TooFewPointsInRing {
         expected_min: i32,
         actual: i32,
@@ -57,7 +63,9 @@ pub enum WkbError {
     ExpectedLineStringInMultiLineString { position: usize },
     #[error("Expected Polygon in MultiPolygon at position {position}")]
     ExpectedPolygonInMultiPolygon { position: usize },
-    #[error("Dimension mismatch at position {position}: expected Z={expected_has_z}, M={expected_has_m}, got Z={actual_has_z}, M={actual_has_m}")]
+    #[error(
+        "Dimension mismatch at position {position}: expected Z={expected_has_z}, M={expected_has_m}, got Z={actual_has_z}, M={actual_has_m}"
+    )]
     DimensionMismatch {
         expected_has_z: bool,
         actual_has_z: bool,
@@ -454,12 +462,12 @@ impl WkbReader {
             }
         }
 
-        if let (Some(first), Some(last)) = (first_coords, last_coords) {
-            if first != last {
-                return Err(WkbError::RingNotClosed {
-                    position: num_points_pos,
-                });
-            }
+        if let (Some(first), Some(last)) = (first_coords, last_coords)
+            && first != last
+        {
+            return Err(WkbError::RingNotClosed {
+                position: num_points_pos,
+            });
         }
 
         Ok(())
diff --git a/crates/sail-function/src/scalar/hash/spark_murmur3_hash.rs b/crates/sail-function/src/scalar/hash/spark_murmur3_hash.rs
index b752cbb3f..0534592f1 100644
--- a/crates/sail-function/src/scalar/hash/spark_murmur3_hash.rs
+++ b/crates/sail-function/src/scalar/hash/spark_murmur3_hash.rs
@@ -4,7 +4,7 @@ use datafusion::arrow::array::{ArrayRef, Int32Array};
 use datafusion::arrow::datatypes::DataType;
 use datafusion::common::Result;
 use datafusion::logical_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility};
-use datafusion_common::{internal_err, DataFusionError, ScalarValue};
+use datafusion_common::{DataFusionError, ScalarValue, internal_err};
 use datafusion_expr::ScalarFunctionArgs;
 
 use crate::scalar::hash::utils::create_murmur3_hashes;
diff --git a/crates/sail-function/src/scalar/hash/utils.rs b/crates/sail-function/src/scalar/hash/utils.rs
index e82424094..1055abffe 100644
--- a/crates/sail-function/src/scalar/hash/utils.rs
+++ b/crates/sail-function/src/scalar/hash/utils.rs
@@ -3,8 +3,8 @@ use std::sync::Arc;
 use datafusion::arrow::array::*;
 use datafusion::arrow::compute::take;
 use datafusion::arrow::datatypes::{
-    ArrowDictionaryKeyType, ArrowNativeType, ArrowNativeTypeOp, DataType, Int16Type, Int32Type,
-    Int64Type, Int8Type, TimeUnit, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
+    ArrowDictionaryKeyType, ArrowNativeType, ArrowNativeTypeOp, DataType, Int8Type, Int16Type,
+    Int32Type, Int64Type, TimeUnit, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
 };
 use datafusion::error::{DataFusionError, Result};
 
@@ -43,17 +43,19 @@ pub fn spark_compatible_murmur3_hash>(data: T, seed: u32) -> u32
 
     #[inline]
     unsafe fn hash_bytes_by_int(data: &[u8], seed: u32) -> i32 {
-        // safety: data length must be aligned to 4 bytes
-        let mut h1 = seed as i32;
-        for i in (0..data.len()).step_by(4) {
-            let ints = data.as_ptr().add(i) as *const i32;
-            let mut half_word = ints.read_unaligned();
-            if cfg!(target_endian = "big") {
-                half_word = half_word.reverse_bits();
+        unsafe {
+            // safety: data length must be aligned to 4 bytes
+            let mut h1 = seed as i32;
+            for i in (0..data.len()).step_by(4) {
+                let ints = data.as_ptr().add(i) as *const i32;
+                let mut half_word = ints.read_unaligned();
+                if cfg!(target_endian = "big") {
+                    half_word = half_word.reverse_bits();
+                }
+                h1 = mix_h1(h1, mix_k1(half_word));
             }
-            h1 = mix_h1(h1, mix_k1(half_word));
+            h1
         }
-        h1
     }
     let data = data.as_ref();
     let len = data.len();
@@ -387,7 +389,7 @@ macro_rules! create_hashes_internal {
                         return Err(DataFusionError::Internal(format!(
                             "Unsupported dictionary type in hasher hashing: {}",
                             col.data_type(),
-                        )))
+                        )));
                     }
                 },
                 _ => {
@@ -425,13 +427,13 @@ mod tests {
     use std::sync::Arc;
 
     use datafusion::arrow::array::{
-        Array, ArrayRef, Float32Array, Float64Array, Int32Array, Int64Array, Int8Array, StringArray,
+        Array, ArrayRef, Float32Array, Float64Array, Int8Array, Int32Array, Int64Array, StringArray,
     };
 
     use super::create_murmur3_hashes;
 
     macro_rules! test_hashes_internal {
-        ($hash_method: ident, $input: expr, $initial_seeds: expr, $expected: expr) => {
+        ($hash_method: ident, $input: expr_2021, $initial_seeds: expr_2021, $expected: expr_2021) => {
             let i = $input;
             let mut hashes = $initial_seeds.clone();
             $hash_method(&[i], &mut hashes).unwrap();
diff --git a/crates/sail-function/src/scalar/json/common.rs b/crates/sail-function/src/scalar/json/common.rs
index 3291d2b71..86b9ae997 100644
--- a/crates/sail-function/src/scalar/json/common.rs
+++ b/crates/sail-function/src/scalar/json/common.rs
@@ -6,18 +6,18 @@ use std::str::Utf8Error;
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    downcast_array, AnyDictionaryArray, Array, ArrayAccessor, ArrayRef, AsArray, DictionaryArray,
-    LargeStringArray, PrimitiveArray, PrimitiveBuilder, RunArray, StringArray, StringViewArray,
+    AnyDictionaryArray, Array, ArrayAccessor, ArrayRef, AsArray, DictionaryArray, LargeStringArray,
+    PrimitiveArray, PrimitiveBuilder, RunArray, StringArray, StringViewArray, downcast_array,
 };
 use datafusion::arrow::compute::kernels::cast;
 use datafusion::arrow::compute::take;
 use datafusion::arrow::datatypes::{ArrowNativeType, DataType, Int64Type, UInt64Type};
-use datafusion_common::{exec_err, plan_err, DataFusionError, Result, ScalarValue};
+use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, plan_err};
 use datafusion_expr::ColumnarValue;
 use jiter::{Jiter, JiterError, Peek};
 
 use crate::scalar::json::common_union::{
-    is_json_union, json_from_union_scalar, nested_json_array, nested_json_array_ref, TYPE_ID_NULL,
+    TYPE_ID_NULL, is_json_union, json_from_union_scalar, nested_json_array, nested_json_array_ref,
 };
 
 /// General implementation of `ScalarUDFImpl::return_type`.
@@ -39,7 +39,9 @@ pub fn return_type_check(
     let first_dict_key_type = dict_key_type(first);
     if !(is_str(first) || is_json_union(first) || first_dict_key_type.is_some()) {
         // if !matches!(first, DataType::Utf8 | DataType::LargeUtf8) {
-        return plan_err!("Unexpected argument type to '{fn_name}' at position 1, expected a string, got {first:?}.");
+        return plan_err!(
+            "Unexpected argument type to '{fn_name}' at position 1, expected a string, got {first:?}."
+        );
     }
     args.iter().skip(1).enumerate().try_for_each(|(index, arg)| {
         if is_str(arg) || is_int(arg) || dict_key_type(arg).is_some() {
@@ -71,10 +73,10 @@ fn is_int(d: &DataType) -> bool {
 }
 
 fn dict_key_type(d: &DataType) -> Option {
-    if let DataType::Dictionary(key, value) = d {
-        if is_str(value) || is_json_union(value) {
-            return Some(*key.clone());
-        }
+    if let DataType::Dictionary(key, value) = d
+        && (is_str(value) || is_json_union(value))
+    {
+        return Some(*key.clone());
     }
     None
 }
diff --git a/crates/sail-function/src/scalar/json/common_union.rs b/crates/sail-function/src/scalar/json/common_union.rs
index 13f3c9e56..0cd025a54 100644
--- a/crates/sail-function/src/scalar/json/common_union.rs
+++ b/crates/sail-function/src/scalar/json/common_union.rs
@@ -48,12 +48,12 @@ pub(crate) fn json_from_union_scalar<'a>(
 ) -> Option<&'a str> {
     if let Some((type_id, value)) = type_id_value {
         // we only want to take the ScalarValue string if the type_id indicates the value represents nested JSON
-        if fields == &union_fields() && (*type_id == TYPE_ID_ARRAY || *type_id == TYPE_ID_OBJECT) {
-            if let ScalarValue::Utf8(s) | ScalarValue::Utf8View(s) | ScalarValue::LargeUtf8(s) =
+        if fields == &union_fields()
+            && (*type_id == TYPE_ID_ARRAY || *type_id == TYPE_ID_OBJECT)
+            && let ScalarValue::Utf8(s) | ScalarValue::Utf8View(s) | ScalarValue::LargeUtf8(s) =
                 value.as_ref()
-            {
-                return s.as_deref();
-            }
+        {
+            return s.as_deref();
         }
     }
     None
diff --git a/crates/sail-function/src/scalar/json/from_json.rs b/crates/sail-function/src/scalar/json/from_json.rs
index 3d832d3dd..af07c632f 100644
--- a/crates/sail-function/src/scalar/json/from_json.rs
+++ b/crates/sail-function/src/scalar/json/from_json.rs
@@ -6,7 +6,7 @@ use datafusion::arrow::array::*;
 use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
 use datafusion::arrow::datatypes::*;
 use datafusion::error::{DataFusionError, Result};
-use datafusion_common::{exec_err, plan_err, ScalarValue};
+use datafusion_common::{ScalarValue, exec_err, plan_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
 };
@@ -22,7 +22,7 @@ use serde_json::Value;
 
 use crate::functions_nested_utils::*;
 use crate::functions_utils::make_scalar_function;
-use crate::scalar::datetime::utils::spark_datetime_format_to_chrono_strftime;
+use crate::scalar::datetime::format::DateTimeFormat;
 
 /// UDF implementation of `from_json`, similar to Spark's `from_json`.
 /// This function parses a column of JSON strings using a specified schema
@@ -47,29 +47,37 @@ pub struct SparkFromJson {
 /// Configuration options for the `from_json` function.
 #[derive(Debug)]
 struct SparkFromJsonOptions {
-    timestamp_format: String,
-    date_format: String,
+    timestamp_format: DateTimeFormat,
+    date_format: DateTimeFormat,
 }
 
 impl SparkFromJsonOptions {
     pub const TIMESTAMP_FORMAT_OPTION: &'static str = "timestampFormat";
     pub const DATE_FORMAT_OPTION: &'static str = "dateFormat";
-    // Default formats matching Spark's behavior
-    pub const TIMESTAMP_FORMAT_DEFAULT: &'static str = "%Y-%m-%d %H:%M:%S";
-    pub const DATE_FORMAT_DEFAULT: &'static str = "%Y-%m-%d";
+    // Default formats matching Spark's behavior (Java DateTimeFormatter patterns)
+    pub const TIMESTAMP_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd HH:mm:ss";
+    pub const DATE_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd";
 
     fn from_map(map: &MapArray) -> Result {
         let timestamp_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
-            .unwrap_or_else(|| Self::TIMESTAMP_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_FORMAT_DEFAULT)
+                    .expect("default timestamp format should be valid")
+            });
 
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_parsing)
             .transpose()?
-            .unwrap_or_else(|| Self::DATE_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            });
 
         Ok(Self {
             timestamp_format,
@@ -79,10 +87,13 @@ impl SparkFromJsonOptions {
 }
 
 impl Default for SparkFromJsonOptions {
+    #[expect(clippy::expect_used)]
     fn default() -> Self {
         Self {
-            timestamp_format: Self::TIMESTAMP_FORMAT_DEFAULT.to_string(),
-            date_format: Self::DATE_FORMAT_DEFAULT.to_string(),
+            timestamp_format: DateTimeFormat::for_parsing(Self::TIMESTAMP_FORMAT_DEFAULT)
+                .expect("default timestamp format should be valid"),
+            date_format: DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
+                .expect("default date format should be valid"),
         }
     }
 }
@@ -154,16 +165,19 @@ impl ScalarUDFImpl for SparkFromJson {
 
     fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
         match arg_types {
-            [DataType::Null | DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8, DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8] => {
-                Ok(vec![DataType::Utf8, arg_types[1].clone()])
-            }
-            [DataType::Null | DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8, DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8, DataType::Map(_, _)] => {
-                Ok(vec![
-                    DataType::Utf8,
-                    arg_types[1].clone(),
-                    arg_types[2].clone(),
-                ])
-            }
+            [
+                DataType::Null | DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+            ] => Ok(vec![DataType::Utf8, arg_types[1].clone()]),
+            [
+                DataType::Null | DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Map(_, _),
+            ] => Ok(vec![
+                DataType::Utf8,
+                arg_types[1].clone(),
+                arg_types[2].clone(),
+            ]),
             _ => plan_err!(
                 "`{}` function requires 2 or 3 arguments, got {}",
                 Self::FROM_JSON_NAME,
@@ -216,7 +230,7 @@ fn from_json_inner(args: &[ArrayRef], session_timezone: &str) -> Result Ok(FieldBuilder::LargeString(
-            LargeStringBuilder::with_capacity(capacity, capacity * 16),
+            LargeStringBuilder::with_capacity(capacity, 0),
         )),
         DataType::Map(field, ordered) => {
             let (keys_field, values_field) = match field.data_type() {
-                DataType::Struct(fields) => {
-                    Ok((fields[0].clone(), fields[1].clone()))
-                },
+                DataType::Struct(fields) => Ok((fields[0].clone(), fields[1].clone())),
                 other => exec_err!(
                     "Unreachable: `{}` function handled map field that should be a struct but is {:#?}",
                     SparkFromJson::FROM_JSON_NAME,
                     other
-                )
+                ),
             }?;
             let keys_builder = create_builder(keys_field.data_type(), capacity, session_timezone)?;
             let values_builder =
@@ -420,8 +432,7 @@ fn create_builder(
             })
         }
         DataType::Utf8 => Ok(FieldBuilder::String(StringBuilder::with_capacity(
-            capacity,
-            capacity * 16,
+            capacity, 0,
         ))),
         DataType::Timestamp(time_unit, tz) => {
             let resolved_tz = tz.clone().unwrap_or(Arc::from(session_timezone));
@@ -770,7 +781,7 @@ fn finish_builder(builder: FieldBuilder) -> Result {
                     "Unreachable: `{}` function handled map field that should be a struct but is {:#?}",
                     SparkFromJson::FROM_JSON_NAME,
                     other
-                )
+                ),
             }?;
             let struct_array =
                 StructArray::new(fields.clone(), vec![finished_keys, finished_values], None);
@@ -928,12 +939,8 @@ fn parse_date32(
     options: &SparkFromJsonOptions,
 ) -> Result<::Native> {
     let format = &options.date_format;
-    let naive_date = NaiveDate::parse_from_str(s, format).map_err(|e| {
-        DataFusionError::Execution(format!(
-            "Failed to parse date '{s}' with format '{format}': {e}"
-        ))
-    })?;
-    Ok(Date32Type::from_naive_date(naive_date))
+    let parsed = format.parse_datetime_value(s)?;
+    Ok(Date32Type::from_naive_date(parsed.datetime.date()))
 }
 
 fn parse_timestamp(
@@ -942,21 +949,23 @@ fn parse_timestamp(
     options: &SparkFromJsonOptions,
 ) -> Result> {
     let format = &options.timestamp_format;
-    let naive_datetime = if let Ok(datetime) = NaiveDateTime::parse_from_str(s, format) {
-        datetime
-    } else if let Ok(date) = NaiveDate::parse_from_str(s, format) {
-        let Some(datetime) = date.and_hms_opt(0, 0, 0) else {
-            return exec_err!("Failed to parse timestamp '{s}': invalid date");
-        };
-        datetime
+    let parsed = format.parse_datetime_value(s)?;
+
+    let datetime = if let Some(offset) = parsed.offset {
+        parsed
+            .datetime
+            .and_local_timezone(offset)
+            .single()
+            .map(|x| x.to_utc())
+            .ok_or_else(|| DataFusionError::Execution("cannot apply parsed offset".to_string()))?
     } else {
-        return exec_err!("Failed to parse timestamp '{s}' with format '{format}'");
+        let tz: Tz = timezone.as_ref().parse().map_err(|e| {
+            DataFusionError::Execution(format!("Invalid timezone '{timezone}': {e}"))
+        })?;
+        localize_with_fallback(&tz, &parsed.datetime)?
     };
-    let tz: Tz = timezone
-        .as_ref()
-        .parse()
-        .map_err(|e| DataFusionError::Execution(format!("Invalid timezone '{timezone}': {e}")))?;
-    localize_with_fallback(&tz, &naive_datetime)
+
+    Ok(datetime)
 }
 
 /// Parses a schema string into an Arrow DataType. The schema may be a bare field list
@@ -1124,20 +1133,18 @@ fn json_type_name_to_data_type(type_name: &str, session_timezone: &str) -> Resul
                 let mut parts = args.split(',').map(str::trim);
                 if let (Some(precision), Some(scale), None) =
                     (parts.next(), parts.next(), parts.next())
-                {
-                    if let (Ok(precision), Ok(scale)) =
+                    && let (Ok(precision), Ok(scale)) =
                         (precision.parse::(), scale.parse::())
-                    {
-                        datafusion::arrow::datatypes::validate_decimal_precision_and_scale::<
-                            Decimal128Type,
-                        >(precision, scale)
-                        .map_err(|e| {
-                            DataFusionError::Plan(format!(
-                                "Invalid decimal precision/scale in '{other}': {e}"
-                            ))
-                        })?;
-                        return Ok(DataType::Decimal128(precision, scale));
-                    }
+                {
+                    datafusion::arrow::datatypes::validate_decimal_precision_and_scale::<
+                        Decimal128Type,
+                    >(precision, scale)
+                    .map_err(|e| {
+                        DataFusionError::Plan(format!(
+                            "Invalid decimal precision/scale in '{other}': {e}"
+                        ))
+                    })?;
+                    return Ok(DataType::Decimal128(precision, scale));
                 }
             }
             if let Some(args) = other
diff --git a/crates/sail-function/src/scalar/json/json_as_text.rs b/crates/sail-function/src/scalar/json/json_as_text.rs
index f228c2b04..bc6bc8882 100644
--- a/crates/sail-function/src/scalar/json/json_as_text.rs
+++ b/crates/sail-function/src/scalar/json/json_as_text.rs
@@ -13,7 +13,7 @@ use datafusion_expr::{
 use jiter::Peek;
 
 use crate::scalar::json::common::{
-    get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath,
+    GetError, InvokeResult, JsonPath, get_err, invoke, jiter_json_find, return_type_check,
 };
 
 #[derive(Debug, PartialEq, Eq, Hash)]
diff --git a/crates/sail-function/src/scalar/json/json_length.rs b/crates/sail-function/src/scalar/json/json_length.rs
index 6a7e40d00..c3151fb66 100644
--- a/crates/sail-function/src/scalar/json/json_length.rs
+++ b/crates/sail-function/src/scalar/json/json_length.rs
@@ -13,7 +13,7 @@ use datafusion_expr::{
 use jiter::Peek;
 
 use crate::scalar::json::common::{
-    get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath,
+    GetError, InvokeResult, JsonPath, get_err, invoke, jiter_json_find, return_type_check,
 };
 
 #[derive(Debug, PartialEq, Eq, Hash)]
diff --git a/crates/sail-function/src/scalar/json/json_object_keys.rs b/crates/sail-function/src/scalar/json/json_object_keys.rs
index 405243429..c34901359 100644
--- a/crates/sail-function/src/scalar/json/json_object_keys.rs
+++ b/crates/sail-function/src/scalar/json/json_object_keys.rs
@@ -13,7 +13,7 @@ use datafusion_expr::{
 use jiter::Peek;
 
 use crate::scalar::json::common::{
-    get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath,
+    GetError, InvokeResult, JsonPath, get_err, invoke, jiter_json_find, return_type_check,
 };
 
 #[derive(Debug, PartialEq, Eq, Hash)]
diff --git a/crates/sail-function/src/scalar/json/mod.rs b/crates/sail-function/src/scalar/json/mod.rs
index 17f7f106c..2b596d4e1 100644
--- a/crates/sail-function/src/scalar/json/mod.rs
+++ b/crates/sail-function/src/scalar/json/mod.rs
@@ -8,10 +8,10 @@ mod to_json;
 mod common;
 mod common_union;
 
-pub use common_union::{JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE};
+pub use common_union::{JSON_UNION_DATA_TYPE, JsonUnionEncoder, JsonUnionValue};
 pub use from_json::SparkFromJson;
-pub use json_as_text::{json_as_text_udf, JsonAsText};
-pub use json_length::{json_length_udf, JsonLength};
-pub use json_object_keys::{json_object_keys_udf, JsonObjectKeys};
+pub use json_as_text::{JsonAsText, json_as_text_udf};
+pub use json_length::{JsonLength, json_length_udf};
+pub use json_object_keys::{JsonObjectKeys, json_object_keys_udf};
 pub use schema_of_json::SparkSchemaOfJson;
-pub use to_json::{to_json_udf, SparkToJson};
+pub use to_json::{SparkToJson, to_json_udf};
diff --git a/crates/sail-function/src/scalar/json/schema_of_json.rs b/crates/sail-function/src/scalar/json/schema_of_json.rs
index 95e68a388..6688dd8b1 100644
--- a/crates/sail-function/src/scalar/json/schema_of_json.rs
+++ b/crates/sail-function/src/scalar/json/schema_of_json.rs
@@ -2,17 +2,23 @@ use std::iter::Peekable;
 use std::str::Chars;
 use std::sync::Arc;
 
+use chrono::{NaiveDate, NaiveTime};
+use chrono_tz::Tz;
 use datafusion::arrow::array::{
-    downcast_array, Array, ArrayRef, MapArray, StringArray, StructArray,
+    Array, ArrayRef, MapArray, StringArray, StructArray, downcast_array,
 };
-use datafusion::arrow::datatypes::{DataType, Field, Fields};
-use datafusion_common::{exec_err, plan_err, DataFusionError, Result};
+use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Fields};
+use datafusion_common::{DataFusionError, Result, exec_err, plan_err};
 use datafusion_expr::function::Hint;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature};
+use datafusion_expr::{
+    ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+};
 use datafusion_expr_common::signature::Volatility;
 use datafusion_functions::downcast_arg;
 use datafusion_functions::utils::make_scalar_function;
 
+use crate::schema_inference::{InferredType, TypeMerger};
+
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct SparkSchemaOfJson {
     signature: Signature,
@@ -63,24 +69,25 @@ impl SparkSchemaOfJson {
     fn validate_arg_types(arg_types: &[DataType]) -> Result<()> {
         match arg_types {
             [DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8] => Ok(()),
-            [DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8, DataType::Map(map_field, _)] => {
-                match map_field.data_type() {
-                    DataType::Struct(fields) => {
-                        let key = fields[0].clone();
-                        let value = fields[1].clone();
-                        if !key.data_type().is_string() || !value.data_type().is_string() {
-                            return Err(DataFusionError::Plan(format!(
-                                "For function `{}`, the options map keys/values should both be type string. Instead got key: {}, value: {}",
-                                Self::SCHEMA_OF_JSON_NAME,
-                                key.data_type(),
-                                value.data_type(),
-                            )));
-                        }
-                        Ok(())
+            [
+                DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Map(map_field, _),
+            ] => match map_field.data_type() {
+                DataType::Struct(fields) => {
+                    let key = fields[0].clone();
+                    let value = fields[1].clone();
+                    if !key.data_type().is_string() || !value.data_type().is_string() {
+                        return Err(DataFusionError::Plan(format!(
+                            "For function `{}`, the options map keys/values should both be type string. Instead got key: {}, value: {}",
+                            Self::SCHEMA_OF_JSON_NAME,
+                            key.data_type(),
+                            value.data_type(),
+                        )));
                     }
-                    _ => unreachable!(),
+                    Ok(())
                 }
-            }
+                _ => unreachable!(),
+            },
             _ => plan_err!(
                 "For function `{:?}` found invalid arg types: {:?}",
                 Self::SCHEMA_OF_JSON_NAME,
@@ -103,8 +110,16 @@ impl ScalarUDFImpl for SparkSchemaOfJson {
         Ok(DataType::Utf8)
     }
 
+    fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result {
+        // `schema_of_json` only accepts a foldable, non-null input and always
+        // produces a DDL string, so the result is never null. Spark marks the
+        // output of a successful foldable call as non-nullable.
+        Ok(Arc::new(Field::new(self.name(), DataType::Utf8, false)))
+    }
+
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
-        Self::validate_args_len(&args.args)?;
+        // Arity is validated in `coerce_types` (planning) and defensively in
+        // `schema_of_json_inner` (execution), so no arg-count check here.
         Self::validate_args_are_literal(&args.args)?;
         let hints = vec![Hint::AcceptsSingular, Hint::AcceptsSingular];
         make_scalar_function(schema_of_json_inner, hints)(&args.args)
@@ -146,6 +161,13 @@ fn schema_of_json_inner(args: &[ArrayRef]) -> Result {
         return Err(DataFusionError::Execution(
             "No value passed into input".to_string(),
         ));
+    } else if rows.is_null(0) {
+        // Spark rejects a null `json` argument with a DATATYPE_MISMATCH at
+        // analysis time, since the input must be a foldable, non-null value.
+        return plan_err!(
+            "For function `{}`, the json must not be null",
+            SparkSchemaOfJson::SCHEMA_OF_JSON_NAME
+        );
     } else if rows.value(0).is_empty() {
         "STRING".to_string()
     } else {
@@ -181,23 +203,6 @@ fn infer_json_schema_type(json_string: &str, options: &SparkSchemaOfJsonOptions)
 /// The maximum precision of Spark's `DecimalType`.
 const MAX_DECIMAL_PRECISION: usize = 38;
 
-/// A JSON type inferred from a literal JSON string, mirroring the types that
-/// Spark's `JsonInferSchema` can produce for `schema_of_json`.
-#[derive(Debug, Clone, PartialEq)]
-enum InferredType {
-    Null,
-    Boolean,
-    Long,
-    /// A number inferred as a decimal with the given precision and scale.
-    Decimal(u8, u8),
-    Double,
-    String,
-    Array(Box),
-    /// Fields are sorted by name and duplicate names are preserved, matching
-    /// Spark's `JsonInferSchema`.
-    Struct(Vec<(String, InferredType)>),
-}
-
 /// A parser that infers the Spark type of a literal JSON string, mirroring
 /// the Jackson lexing behavior that Spark relies on for `schema_of_json`,
 /// including features that strict JSON parsers reject: single-quoted strings,
@@ -247,20 +252,20 @@ impl<'a> JsonSchemaParser<'a> {
             Some('{') => self.parse_object(),
             Some('[') => self.parse_array(),
             Some('"') => {
-                self.parse_string('"')?;
-                Ok(InferredType::String)
+                let value = self.parse_string('"')?;
+                Ok(self.infer_string_type(&value))
             }
             Some('\'') if self.options.allow_single_quotes => {
-                self.parse_string('\'')?;
-                Ok(InferredType::String)
+                let value = self.parse_string('\'')?;
+                Ok(self.infer_string_type(&value))
             }
             Some('t') => {
                 self.expect_literal("true")?;
-                Ok(InferredType::Boolean)
+                Ok(self.as_primitive(InferredType::Boolean))
             }
             Some('f') => {
                 self.expect_literal("false")?;
-                Ok(InferredType::Boolean)
+                Ok(self.as_primitive(InferredType::Boolean))
             }
             Some('n') => {
                 self.expect_literal("null")?;
@@ -268,29 +273,56 @@ impl<'a> JsonSchemaParser<'a> {
             }
             Some('N') if self.options.allow_non_numeric_numbers => {
                 self.expect_literal("NaN")?;
-                Ok(InferredType::Double)
+                Ok(self.as_primitive(InferredType::Double))
             }
             Some('I') if self.options.allow_non_numeric_numbers => {
                 self.expect_literal("Infinity")?;
-                Ok(InferredType::Double)
+                Ok(self.as_primitive(InferredType::Double))
             }
             Some(sign @ ('-' | '+')) => {
                 self.chars.next();
                 if self.chars.peek() == Some(&'I') && self.options.allow_non_numeric_numbers {
                     self.expect_literal("Infinity")?;
-                    Ok(InferredType::Double)
+                    Ok(self.as_primitive(InferredType::Double))
                 } else if sign == '-' {
-                    self.parse_number(true)
+                    let number = self.parse_number(true)?;
+                    Ok(self.as_primitive(number))
                 } else {
                     // A `+` sign is only valid before `Infinity`.
                     exec_err!("unexpected character `+` when parsing a JSON value")
                 }
             }
-            Some(c) if c.is_ascii_digit() => self.parse_number(false),
+            Some(c) if c.is_ascii_digit() => {
+                let number = self.parse_number(false)?;
+                Ok(self.as_primitive(number))
+            }
             Some(c) => exec_err!("unexpected character `{c}` when parsing a JSON value"),
         }
     }
 
+    /// Coerces a scalar primitive to `STRING` when the `primitivesAsString`
+    /// option is enabled, matching Spark's `JsonInferSchema`. Strings, structs,
+    /// and arrays are unaffected.
+    fn as_primitive(&self, inferred: InferredType) -> InferredType {
+        if self.options.primitives_as_string {
+            InferredType::String
+        } else {
+            inferred
+        }
+    }
+
+    /// Infers the type of a parsed JSON string: `TIMESTAMP` when the
+    /// `inferTimestamp` option is enabled and the value matches a recognized
+    /// timestamp/date pattern, otherwise `STRING`. Spark infers timestamps from
+    /// strings independently of `primitivesAsString`.
+    fn infer_string_type(&self, value: &str) -> InferredType {
+        if self.options.infer_timestamp && is_timestamp_string(value) {
+            InferredType::Timestamp
+        } else {
+            InferredType::String
+        }
+    }
+
     fn parse_object(&mut self) -> Result {
         self.expect_char('{')?;
         let mut fields: Vec<(String, InferredType)> = Vec::new();
@@ -308,7 +340,7 @@ impl<'a> JsonSchemaParser<'a> {
                     self.parse_unquoted_name()
                 }
                 Some(c) => {
-                    return exec_err!("unexpected character `{c}` when parsing a field name")
+                    return exec_err!("unexpected character `{c}` when parsing a field name");
                 }
                 None => return exec_err!("unexpected end of input when parsing a field name"),
             };
@@ -364,7 +396,7 @@ impl<'a> JsonSchemaParser<'a> {
                 Some(c) if c == quote => return Ok(value),
                 Some('\\') => value.push(self.parse_escape(quote)?),
                 Some(c) if (c as u32) < 0x20 => {
-                    return exec_err!("unescaped control character in a string")
+                    return exec_err!("unescaped control character in a string");
                 }
                 Some(c) => value.push(c),
             }
@@ -535,21 +567,268 @@ fn is_unquoted_name_char(c: char) -> bool {
         || !c.is_ascii()
 }
 
+/// The parsed pieces of a timestamp string: nine integer segments
+/// (year, month, day, hour, minute, second, microsecond, and two spare zone
+/// segments), an optional timezone string, and whether only a time was given.
+struct ParsedTimestamp {
+    segments: [i64; 9],
+    timezone: Option,
+    just_time: bool,
+}
+
+/// Mirrors Spark's `isValidDigits`: the year segment allows 4-6 digits, the
+/// microsecond segment any number of digits, the zone segment 0-2 digits, and
+/// every other segment 1-2 digits.
+fn is_valid_timestamp_digits(segment: usize, digits: usize) -> bool {
+    const MAX_DIGITS_YEAR: usize = 6;
+    segment == 6
+        || (segment == 0 && (4..=MAX_DIGITS_YEAR).contains(&digits))
+        || (segment == 7 && digits <= 2)
+        || (segment != 0 && segment != 6 && segment != 7 && (1..=2).contains(&digits))
+}
+
+/// Spark trims leading/trailing whitespace and ISO control characters.
+fn is_whitespace_or_iso_control(b: u8) -> bool {
+    b <= b' ' || b == 0x7f
+}
+
+/// A faithful port of Spark's `SparkDateTimeUtils.parseTimestampString`. Splits
+/// the input into timestamp segments using Spark's exact delimiter rules, or
+/// returns `None` if the string is not shaped like a timestamp.
+fn parse_timestamp_string(s: &str) -> Option {
+    let bytes = s.as_bytes();
+    let mut segments: [i64; 9] = [1, 1, 1, 0, 0, 0, 0, 0, 0];
+    let mut i: usize = 0;
+    let mut current_value: i64 = 0;
+    let mut current_digits: usize = 0;
+
+    let mut start = 0;
+    while start < bytes.len() && is_whitespace_or_iso_control(bytes[start]) {
+        start += 1;
+    }
+    let mut str_end = bytes.len();
+    while str_end > start && is_whitespace_or_iso_control(bytes[str_end - 1]) {
+        str_end -= 1;
+    }
+    if start == str_end {
+        return None;
+    }
+
+    let mut digits_milli = 0usize;
+    let mut just_time = false;
+    let mut timezone: Option = None;
+    let mut year_sign: Option = None;
+    let mut j = start;
+    if bytes[j] == b'-' || bytes[j] == b'+' {
+        year_sign = Some(if bytes[j] == b'-' { -1 } else { 1 });
+        j += 1;
+    }
+
+    while j < str_end {
+        let b = bytes[j];
+        if b.is_ascii_digit() {
+            let parsed = i64::from(b - b'0');
+            if i == 6 {
+                digits_milli += 1;
+            }
+            // Truncate the fractional part beyond six digits, matching Spark.
+            if i != 6 || current_digits < 6 {
+                current_value = current_value * 10 + parsed;
+            }
+            current_digits += 1;
+        } else if j == 0 && b == b'T' {
+            just_time = true;
+            i += 3;
+        } else if i < 2 {
+            if b == b'-' {
+                if !is_valid_timestamp_digits(i, current_digits) {
+                    return None;
+                }
+                segments[i] = current_value;
+                current_value = 0;
+                current_digits = 0;
+                i += 1;
+            } else if i == 0 && b == b':' && year_sign.is_none() {
+                just_time = true;
+                if !is_valid_timestamp_digits(3, current_digits) {
+                    return None;
+                }
+                segments[3] = current_value;
+                current_value = 0;
+                current_digits = 0;
+                i = 4;
+            } else {
+                return None;
+            }
+        } else if i == 2 {
+            if b == b' ' || b == b'T' {
+                if !is_valid_timestamp_digits(i, current_digits) {
+                    return None;
+                }
+                segments[i] = current_value;
+                current_value = 0;
+                current_digits = 0;
+                i += 1;
+            } else {
+                return None;
+            }
+        } else if i == 3 || i == 4 {
+            if b == b':' {
+                if !is_valid_timestamp_digits(i, current_digits) {
+                    return None;
+                }
+                segments[i] = current_value;
+                current_value = 0;
+                current_digits = 0;
+                i += 1;
+            } else {
+                return None;
+            }
+        } else if i == 5 || i == 6 {
+            if b == b'.' && i == 5 {
+                if !is_valid_timestamp_digits(i, current_digits) {
+                    return None;
+                }
+                segments[i] = current_value;
+                current_value = 0;
+                current_digits = 0;
+                i += 1;
+            } else {
+                if !is_valid_timestamp_digits(i, current_digits) {
+                    return None;
+                }
+                segments[i] = current_value;
+                current_value = 0;
+                current_digits = 0;
+                i += 1;
+                timezone = Some(String::from_utf8_lossy(&bytes[j..str_end]).into_owned());
+                j = str_end - 1;
+            }
+            if i == 6 && b != b'.' {
+                i += 1;
+            }
+        } else if i < segments.len() && (b == b':' || b == b' ') {
+            if !is_valid_timestamp_digits(i, current_digits) {
+                return None;
+            }
+            segments[i] = current_value;
+            current_value = 0;
+            current_digits = 0;
+            i += 1;
+        } else {
+            return None;
+        }
+        j += 1;
+    }
+
+    if i >= segments.len() || !is_valid_timestamp_digits(i, current_digits) {
+        return None;
+    }
+    segments[i] = current_value;
+
+    while digits_milli < 6 {
+        segments[6] *= 10;
+        digits_milli += 1;
+    }
+
+    segments[0] *= year_sign.unwrap_or(1);
+    Some(ParsedTimestamp {
+        segments,
+        timezone,
+        just_time,
+    })
+}
+
+/// Validates a timezone string the way Spark's `getZoneId` does for the cases
+/// that appear in JSON: `Z`, numeric offsets (`+02:00`, `-0800`), and IANA
+/// region IDs (`America/Los_Angeles`).
+fn is_valid_timezone(tz: &str) -> bool {
+    let tz = tz.trim();
+    if tz.is_empty() {
+        return false;
+    }
+    if tz == "Z" || tz.eq_ignore_ascii_case("UTC") || tz.eq_ignore_ascii_case("GMT") {
+        return true;
+    }
+    if let Some(offset) = tz.strip_prefix(['+', '-']) {
+        let digits: String = offset.chars().filter(|c| *c != ':').collect();
+        if digits.len() != offset.len() - offset.matches(':').count()
+            || !digits.bytes().all(|b| b.is_ascii_digit())
+        {
+            return false;
+        }
+        let parse2 = |s: &str| s.parse::().ok();
+        let (hours, minutes) = match digits.len() {
+            2 => (parse2(&digits), Some(0)),
+            4 => (parse2(&digits[0..2]), parse2(&digits[2..4])),
+            6 => (parse2(&digits[0..2]), parse2(&digits[2..4])),
+            _ => (None, None),
+        };
+        return match (hours, minutes) {
+            (Some(h), Some(m)) => h <= 18 && m < 60,
+            _ => false,
+        };
+    }
+    tz.parse::().is_ok()
+}
+
+/// Returns true if the string matches a timestamp/date that Spark's
+/// `inferTimestamp` option would promote to `TIMESTAMP`. This is a faithful
+/// port of Spark's `SparkDateTimeUtils.stringToTimestamp` (used for the default
+/// `TIMESTAMP_LTZ` type), including year-only and time-only forms, fractional
+/// seconds, and timezone suffixes, with range validation via `chrono`.
+fn is_timestamp_string(s: &str) -> bool {
+    let Some(parsed) = parse_timestamp_string(s) else {
+        return false;
+    };
+    let (Ok(hour), Ok(minute), Ok(second)) = (
+        u32::try_from(parsed.segments[3]),
+        u32::try_from(parsed.segments[4]),
+        u32::try_from(parsed.segments[5]),
+    ) else {
+        return false;
+    };
+    let micros = u32::try_from(parsed.segments[6]).unwrap_or(u32::MAX);
+    if NaiveTime::from_hms_nano_opt(hour, minute, second, micros.saturating_mul(1000)).is_none() {
+        return false;
+    }
+    if !parsed.just_time {
+        let (Ok(year), Ok(month), Ok(day)) = (
+            i32::try_from(parsed.segments[0]),
+            u32::try_from(parsed.segments[1]),
+            u32::try_from(parsed.segments[2]),
+        ) else {
+            return false;
+        };
+        if NaiveDate::from_ymd_opt(year, month, day).is_none() {
+            return false;
+        }
+    }
+    match &parsed.timezone {
+        Some(tz) => is_valid_timezone(tz),
+        None => true,
+    }
+}
+
 /// Returns the most specific type that both types can be promoted to,
 /// mirroring Spark's `JsonInferSchema.compatibleType`.
 fn merge_types(left: InferredType, right: InferredType) -> InferredType {
-    use InferredType::*;
-    match (left, right) {
-        (Null, t) | (t, Null) => t,
-        (l, r) if l == r => l,
-        (Long, Double) | (Double, Long) => Double,
-        // A long is at most `DECIMAL(20, 0)` when promoted to a decimal.
-        (Long, Decimal(p, s)) | (Decimal(p, s), Long) => merge_decimals(p.max(20), s, 20, 0),
-        (Double, Decimal(_, _)) | (Decimal(_, _), Double) => Double,
-        (Decimal(p1, s1), Decimal(p2, s2)) => merge_decimals(p1, s1, p2, s2),
-        (Array(l), Array(r)) => Array(Box::new(merge_types(*l, *r))),
-        (Struct(l), Struct(r)) => Struct(merge_fields(l, r)),
-        _ => String,
+    left.merge_with(right, &JsonTypeMerger)
+}
+
+struct JsonTypeMerger;
+
+impl TypeMerger for JsonTypeMerger {
+    fn merge_atomic(&self, left: InferredType, right: InferredType) -> InferredType {
+        use InferredType::*;
+        match (left, right) {
+            (Long, Double) | (Double, Long) => Double,
+            // A long is at most `DECIMAL(20, 0)` when promoted to a decimal.
+            (Long, Decimal(p, s)) | (Decimal(p, s), Long) => merge_decimals(p.max(20), s, 20, 0),
+            (Double, Decimal(_, _)) | (Decimal(_, _), Double) => Double,
+            (Decimal(p1, s1), Decimal(p2, s2)) => merge_decimals(p1, s1, p2, s2),
+            _ => String,
+        }
     }
 }
 
@@ -566,27 +845,6 @@ fn merge_decimals(p1: u8, s1: u8, p2: u8, s2: u8) -> InferredType {
     }
 }
 
-/// Merges the fields of two structs by name, mirroring Spark's behavior of
-/// grouping fields by name and reducing each group with `compatibleType`.
-fn merge_fields(
-    left: Vec<(String, InferredType)>,
-    right: Vec<(String, InferredType)>,
-) -> Vec<(String, InferredType)> {
-    let mut fields = left;
-    fields.extend(right);
-    fields.sort_by(|(a, _), (b, _)| a.cmp(b));
-    let mut merged: Vec<(String, InferredType)> = Vec::new();
-    for (name, t) in fields {
-        match merged.last_mut() {
-            Some((last_name, last_type)) if *last_name == name => {
-                *last_type = merge_types(last_type.clone(), t);
-            }
-            _ => merged.push((name, t)),
-        }
-    }
-    merged
-}
-
 /// Writes the type as a Spark DDL string, returning `None` for types that
 /// Spark drops during canonicalization (empty structs, fields with empty
 /// names, and arrays of dropped types). `NULL` types become `STRING`.
@@ -595,9 +853,14 @@ fn canonicalize_ddl(t: &InferredType) -> Option {
         InferredType::Null => Some("STRING".to_string()),
         InferredType::Boolean => Some("BOOLEAN".to_string()),
         InferredType::Long => Some("BIGINT".to_string()),
+        InferredType::Float => Some("FLOAT".to_string()),
         InferredType::Decimal(p, s) => Some(format!("DECIMAL({p},{s})")),
         InferredType::Double => Some("DOUBLE".to_string()),
         InferredType::String => Some("STRING".to_string()),
+        InferredType::Binary => Some("BINARY".to_string()),
+        InferredType::Date => Some("DATE".to_string()),
+        InferredType::Timestamp => Some("TIMESTAMP".to_string()),
+        InferredType::TimestampNtz => Some("TIMESTAMP_NTZ".to_string()),
         InferredType::Array(element) => canonicalize_ddl(element).map(|e| format!("ARRAY<{e}>")),
         InferredType::Struct(fields) => {
             let fields = fields
@@ -613,6 +876,7 @@ fn canonicalize_ddl(t: &InferredType) -> Option {
                 Some(format!("STRUCT<{}>", fields.join(", ")))
             }
         }
+        InferredType::Variant => Some("VARIANT".to_string()),
     }
 }
 
@@ -642,12 +906,14 @@ enum ModeOptions {
 }
 
 impl ModeOptions {
-    fn from_str(value: String) -> Result {
-        match value.as_str() {
-            "PERMISSIVE" => Ok(ModeOptions::Permissive),
-            "FAILFAST" => Ok(ModeOptions::FailFast),
-            "DROPMALFORMED" => Ok(ModeOptions::DropMalformed),
-            other => plan_err!("Invalid mode option: {other}"),
+    /// Mirrors Spark's `ParseMode.fromString`: the value is matched
+    /// case-insensitively and an unrecognized mode falls back to `PERMISSIVE`
+    /// instead of erroring.
+    fn from_str(value: &str) -> Self {
+        match value.to_uppercase().as_str() {
+            "FAILFAST" => ModeOptions::FailFast,
+            "DROPMALFORMED" => ModeOptions::DropMalformed,
+            _ => ModeOptions::Permissive,
         }
     }
 }
@@ -660,6 +926,8 @@ struct SparkSchemaOfJsonOptions {
     allow_non_numeric_numbers: bool,
     prefers_decimal: bool,
     allow_numeric_leading_zeros: bool,
+    primitives_as_string: bool,
+    infer_timestamp: bool,
 }
 
 impl Default for SparkSchemaOfJsonOptions {
@@ -673,6 +941,8 @@ impl Default for SparkSchemaOfJsonOptions {
             allow_non_numeric_numbers: true,
             prefers_decimal: false,
             allow_numeric_leading_zeros: false,
+            primitives_as_string: false,
+            infer_timestamp: false,
         }
     }
 }
@@ -687,7 +957,7 @@ impl SparkSchemaOfJsonOptions {
             let (key, value) = Self::unwrap_or_key_value(key, value)?;
             // Spark reads JSON options through a case-insensitive map.
             match key.to_lowercase().as_str() {
-                "mode" => self.mode = ModeOptions::from_str(value.to_string())?,
+                "mode" => self.mode = ModeOptions::from_str(value),
                 "allowunquotedfieldnames" => {
                     self.allow_unquoted_field_names = Self::parse_boolean_option(key, value)?;
                 }
@@ -703,6 +973,12 @@ impl SparkSchemaOfJsonOptions {
                 "allownumericleadingzeros" => {
                     self.allow_numeric_leading_zeros = Self::parse_boolean_option(key, value)?;
                 }
+                "primitivesasstring" => {
+                    self.primitives_as_string = Self::parse_boolean_option(key, value)?;
+                }
+                "infertimestamp" => {
+                    self.infer_timestamp = Self::parse_boolean_option(key, value)?;
+                }
                 // TODO: support the remaining Spark JSON options below
                 //
                 // These options change the inferred schema or the set of
@@ -710,12 +986,10 @@ impl SparkSchemaOfJsonOptions {
                 // producing a result that diverges from Spark. They all
                 // default to false in Spark, so an explicit false is a no-op
                 // and falls through to the ignored-options arm below.
-                "primitivesasstring"
-                | "allowcomments"
+                "allowcomments"
                 | "allowbackslashescapinganycharacter"
                 | "allowunquotedcontrolchars"
                 | "dropfieldifallnull"
-                | "infertimestamp"
                     if Self::parse_boolean_option(key, value)? =>
                 {
                     return Err(DataFusionError::NotImplemented(format!(
@@ -758,16 +1032,15 @@ impl SparkSchemaOfJsonOptions {
                 } else {
                     return Err(DataFusionError::Plan(format!(
                         "Expected options to be type map but found key type {:?} and value type {:?}",
-                        key_type,
-                        value_type
-                    )))
+                        key_type, value_type
+                    )));
                 }
-            },
+            }
             other => {
                 return Err(DataFusionError::Plan(format!(
                     "Should be unreachable: options should be a map with an inner struct but instead got {:?}",
                     other
-                )))
+                )));
             }
         };
         Ok((keys, values))
@@ -786,3 +1059,58 @@ impl SparkSchemaOfJsonOptions {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::is_timestamp_string;
+
+    #[test]
+    fn timestamp_inference_matches_spark() {
+        // (input, expected) pairs verified against Spark 4.x JVM
+        // (`schema_of_json(..., map('inferTimestamp','true'))`).
+        let cases: &[(&str, bool)] = &[
+            ("2024", true),
+            ("2024-01", true),
+            ("2024-01-02", true),
+            ("2024-1-2", true),
+            ("2024-01-02 03", true),
+            ("2024-01-02 03:04", true),
+            ("2024-01-02 03:04:05", true),
+            ("2024-01-02T03:04:05", true),
+            ("2024-01-02T03:04:05.1", true),
+            ("2024-01-02 03:04:05.123", true),
+            ("2024-01-02 03:04:05.123456789", true),
+            ("2024-01-02T03:04:05.123456", true),
+            ("2024-01-02T03:04:05Z", true),
+            ("2024-01-02 03:04:05+02:00", true),
+            ("2024-01-02 03:04:05-0800", true),
+            ("2024-01-02 03:04:05 America/Los_Angeles", true),
+            ("03:04:05", true),
+            ("03:04", true),
+            ("3:4:5", true),
+            ("T03:04:05", true),
+            (" 2024-01-02 03:04:05 ", true),
+            ("-2024-01-02", true),
+            ("2024-01-02 03:04:05.", true),
+            // Rejected by Spark (inferred as STRING):
+            ("2024-13-02", false),
+            ("2024-01-32", false),
+            ("2024/01/02", false),
+            ("20240102", false),
+            ("2024-01-02x", false),
+            ("2024-01-02  03:04:05", false),
+            ("2024-01-02 25:00:00", false),
+            ("not-a-date", false),
+            ("hello", false),
+            ("", false),
+            ("2024-01-02 03:04:05 Foo/Bar", false),
+        ];
+        for (input, expected) in cases {
+            assert_eq!(
+                is_timestamp_string(input),
+                *expected,
+                "mismatch for input {input:?}"
+            );
+        }
+    }
+}
diff --git a/crates/sail-function/src/scalar/json/to_json.rs b/crates/sail-function/src/scalar/json/to_json.rs
index 7cc5bc7df..e63c58ad5 100644
--- a/crates/sail-function/src/scalar/json/to_json.rs
+++ b/crates/sail-function/src/scalar/json/to_json.rs
@@ -1,13 +1,13 @@
 use std::sync::{Arc, OnceLock};
 
-use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
+use base64::prelude::BASE64_STANDARD;
 use chrono::{TimeZone, Utc};
 use datafusion::arrow::array::{
     Array, ArrayRef, AsArray, BinaryArray, BinaryViewArray, BooleanArray, Date32Array,
-    FixedSizeBinaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
-    Int8Array, LargeListArray, LargeStringArray, ListArray, MapArray, StringArray, StringBuilder,
-    StringViewArray, StructArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+    FixedSizeBinaryArray, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array,
+    Int64Array, LargeListArray, LargeStringArray, ListArray, MapArray, StringArray, StringBuilder,
+    StringViewArray, StructArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array,
 };
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::{Result, ScalarValue};
@@ -20,11 +20,11 @@ use serde_json::{Map, Value};
 
 use crate::functions_nested_utils::opt_downcast_arg;
 use crate::functions_utils::make_scalar_function;
-use crate::scalar::datetime::utils::spark_datetime_format_to_chrono_strftime;
+use crate::scalar::datetime::format::DateTimeFormat;
 
 /// Macro to simplify downcasting arrays and extracting values as JSON
 macro_rules! downcast_and_convert {
-    ($array:expr, $index:expr, $array_type:ty, $convert:expr) => {{
+    ($array:expr_2021, $index:expr_2021, $array_type:ty, $convert:expr_2021) => {{
         let arr = $array
             .as_any()
             .downcast_ref::<$array_type>()
@@ -41,30 +41,39 @@ macro_rules! downcast_and_convert {
 /// Options for to_json function
 #[derive(Debug, PartialEq, Eq, Hash)]
 struct ToJsonOptions {
-    timestamp_format: String,
-    date_format: String,
+    timestamp_format: DateTimeFormat,
+    date_format: DateTimeFormat,
 }
 
 impl ToJsonOptions {
     pub const TIMESTAMP_FORMAT_OPTION: &'static str = "timestampFormat";
     pub const DATE_FORMAT_OPTION: &'static str = "dateFormat";
     // Default ISO 8601 format with timezone offset (not Z)
-    pub const TIMESTAMP_FORMAT_DEFAULT: &'static str = "%Y-%m-%dT%H:%M:%S%.6f%:z";
-    pub const DATE_FORMAT_DEFAULT: &'static str = "%Y-%m-%d";
+    // Using Java DateTimeFormatter patterns
+    pub const TIMESTAMP_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX";
+    pub const DATE_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd";
 
     /// Build ToJsonOptions from a DataFusion MapArray of key-value pairs.
     fn from_map(map: &MapArray) -> Result {
         let timestamp_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
-            .unwrap_or_else(|| Self::TIMESTAMP_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_formatting(Self::TIMESTAMP_FORMAT_DEFAULT)
+                    .expect("default timestamp format should be valid")
+            });
 
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
-            .unwrap_or_else(|| Self::DATE_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            });
 
         Ok(Self {
             timestamp_format,
@@ -74,10 +83,13 @@ impl ToJsonOptions {
 }
 
 impl Default for ToJsonOptions {
+    #[expect(clippy::expect_used)]
     fn default() -> Self {
         Self {
-            timestamp_format: Self::TIMESTAMP_FORMAT_DEFAULT.to_string(),
-            date_format: Self::DATE_FORMAT_DEFAULT.to_string(),
+            timestamp_format: DateTimeFormat::for_formatting(Self::TIMESTAMP_FORMAT_DEFAULT)
+                .expect("default timestamp format should be valid"),
+            date_format: DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
+                .expect("default date format should be valid"),
         }
     }
 }
@@ -126,26 +138,24 @@ impl ScalarUDFImpl for SparkToJson {
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
         // If input is a Variant struct, use the shared variant-to-JSON conversion
         // (Spark's to_json supports Variant input and ignores options for it)
-        if let Some(field) = args.arg_fields.first() {
-            if matches!(field.data_type(), DataType::Struct(_))
-                && crate::scalar::variant::utils::helper::try_field_as_variant_array(field).is_ok()
-            {
-                let result =
-                    crate::scalar::variant::spark_variant_to_json::variant_to_json_columnar(
-                        &args.args[0],
-                    )?;
-                // variant_to_json_columnar returns Utf8View, but to_json promises Utf8
-                return match result {
-                    ColumnarValue::Scalar(ScalarValue::Utf8View(v)) => {
-                        Ok(ColumnarValue::Scalar(ScalarValue::Utf8(v)))
-                    }
-                    ColumnarValue::Array(arr) => Ok(ColumnarValue::Array(arrow::compute::cast(
-                        &arr,
-                        &DataType::Utf8,
-                    )?)),
-                    other => Ok(other),
-                };
-            }
+        if let Some(field) = args.arg_fields.first()
+            && matches!(field.data_type(), DataType::Struct(_))
+            && crate::scalar::variant::utils::helper::try_field_as_variant_array(field).is_ok()
+        {
+            let result = crate::scalar::variant::spark_variant_to_json::variant_to_json_columnar(
+                &args.args[0],
+            )?;
+            // variant_to_json_columnar returns Utf8View, but to_json promises Utf8
+            return match result {
+                ColumnarValue::Scalar(ScalarValue::Utf8View(v)) => {
+                    Ok(ColumnarValue::Scalar(ScalarValue::Utf8(v)))
+                }
+                ColumnarValue::Array(arr) => Ok(ColumnarValue::Array(arrow::compute::cast(
+                    &arr,
+                    &DataType::Utf8,
+                )?)),
+                other => Ok(other),
+            };
         }
         make_scalar_function(to_json_inner, vec![])(&args.args)
     }
@@ -186,7 +196,7 @@ fn to_json_inner(args: &[ArrayRef]) -> Result {
 }
 
 fn array_to_json_strings(array: &ArrayRef, options: &ToJsonOptions) -> Result {
-    let mut builder = StringBuilder::with_capacity(array.len(), array.len() * 64);
+    let mut builder = StringBuilder::with_capacity(array.len(), array.get_buffer_memory_size());
 
     for i in 0..array.len() {
         if array.is_null(i) {
@@ -474,27 +484,60 @@ fn struct_to_values_array(
     Ok(Value::Array(json_values))
 }
 
-fn format_timestamp(value: i64, tz: Option<&str>, format: &str) -> String {
-    if let Some(tz_str) = tz {
-        // Try to parse the timezone and format with offset
-        if let Ok(tz) = tz_str.parse::() {
-            if let Some(dt_utc) = Utc.timestamp_micros(value).single() {
+fn format_timestamp(value: i64, tz: Option<&str>, format: &DateTimeFormat) -> String {
+    use chrono::Offset;
+
+    use crate::scalar::datetime::format::{
+        DateTimeFormatInput, TimePrecision, TimeZoneDisplay, TimestampKind,
+    };
+
+    if let Some(dt_utc) = Utc.timestamp_micros(value).single() {
+        let (datetime, timezone) = if let Some(tz_str) = tz {
+            if let Ok(tz) = tz_str.parse::() {
                 let local_dt = dt_utc.with_timezone(&tz);
-                return local_dt.format(format).to_string();
+                let offset = local_dt.offset().fix();
+                (
+                    local_dt.naive_local(),
+                    Some(TimeZoneDisplay {
+                        offset,
+                        name: Some(tz_str),
+                    }),
+                )
+            } else {
+                (dt_utc.naive_utc(), None)
             }
-        }
-    }
+        } else {
+            (dt_utc.naive_utc(), None)
+        };
+
+        let input = DateTimeFormatInput {
+            datetime,
+            timezone,
+            zone_id: tz,
+            timestamp_kind: TimestampKind::Normal,
+            precision: TimePrecision::Microsecond,
+        };
 
-    // Fallback to UTC
-    Utc.timestamp_micros(value)
-        .single()
-        .map(|ts| ts.format(format).to_string())
-        .unwrap_or_else(|| value.to_string())
+        format.format(input).unwrap_or_else(|_| value.to_string())
+    } else {
+        value.to_string()
+    }
 }
 
-fn format_date(days: i32, format: &str) -> String {
+fn format_date(days: i32, format: &DateTimeFormat) -> String {
+    use crate::scalar::datetime::format::{DateTimeFormatInput, TimePrecision, TimestampKind};
+
     chrono::DateTime::from_timestamp(days as i64 * 24 * 3600, 0)
-        .map(|date| date.format(format).to_string())
+        .map(|date| {
+            let input = DateTimeFormatInput {
+                datetime: date.naive_utc(),
+                timezone: None,
+                zone_id: None,
+                timestamp_kind: TimestampKind::Normal,
+                precision: TimePrecision::Second,
+            };
+            format.format(input).unwrap_or_else(|_| days.to_string())
+        })
         .unwrap_or_else(|| days.to_string())
 }
 
diff --git a/crates/sail-function/src/scalar/map/map_entries.rs b/crates/sail-function/src/scalar/map/map_entries.rs
index 5d56e679a..5be6cf0cd 100644
--- a/crates/sail-function/src/scalar/map/map_entries.rs
+++ b/crates/sail-function/src/scalar/map/map_entries.rs
@@ -4,7 +4,7 @@ use arrow::array::{Array, ArrayRef, ListArray};
 use arrow::datatypes::{DataType, Field, FieldRef};
 use datafusion_common::cast::as_map_array;
 use datafusion_common::utils::take_function_args;
-use datafusion_common::{exec_err, internal_err, Result};
+use datafusion_common::{Result, exec_err, internal_err};
 use datafusion_expr::{
     ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
     Signature, TypeSignature, Volatility,
diff --git a/crates/sail-function/src/scalar/map/str_to_map.rs b/crates/sail-function/src/scalar/map/str_to_map.rs
index 45e13c72e..6a72e304a 100644
--- a/crates/sail-function/src/scalar/map/str_to_map.rs
+++ b/crates/sail-function/src/scalar/map/str_to_map.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{Array, ArrayRef, AsArray, ListArray, StringArray};
 use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion_common::utils::take_function_args;
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{Result, internal_err};
 use datafusion_expr::function::Hint;
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
@@ -15,7 +15,7 @@ use sail_common::spec::{SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME};
 use crate::scalar::map::utils::{
     map_from_keys_values_offsets_nulls, map_type_from_key_value_types,
 };
-use crate::scalar::string::spark_split::{parse_regex, split_to_array, SparkSplit};
+use crate::scalar::string::spark_split::{SparkSplit, parse_regex, split_to_array};
 
 /// Spark-compatible `str_to_map` expression
 /// 
diff --git a/crates/sail-function/src/scalar/map/utils.rs b/crates/sail-function/src/scalar/map/utils.rs
index 7cb68b9e3..37bb87b79 100644
--- a/crates/sail-function/src/scalar/map/utils.rs
+++ b/crates/sail-function/src/scalar/map/utils.rs
@@ -1,12 +1,11 @@
 use std::borrow::Cow;
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use datafusion::arrow::array::{Array, ArrayRef, AsArray, BooleanBuilder, MapArray, StructArray};
 use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer};
 use datafusion::arrow::compute::filter;
 use datafusion::arrow::datatypes::{DataType, Field, Fields};
-use datafusion_common::{exec_err, internal_err, Result, ScalarValue};
+use datafusion_common::{HashSet, Result, ScalarValue, exec_err, internal_err};
 use sail_common::spec::{SAIL_MAP_FIELD_NAME, SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME};
 
 /// Helper function to get element [`DataType`]
@@ -48,14 +47,15 @@ pub fn get_list_offsets(array: &ArrayRef) -> Result> {
     match array.data_type() {
         DataType::List(_) => Ok(Cow::Borrowed(array.as_list::().offsets().as_ref())),
         DataType::LargeList(_) => Ok(Cow::Owned(
-            array.as_list::()
+            array
+                .as_list::()
                 .offsets()
                 .iter()
                 .map(|i| *i as i32)
                 .collect::>(),
         )),
         DataType::FixedSizeList(_, size) => Ok(Cow::Owned(
-             (0..=array.len() as i32).map(|i| size * i).collect()
+            (0..=array.len() as i32).map(|i| size * i).collect(),
         )),
         wrong_type => internal_err!(
             "map_from_arrays expects List/LargeList/FixedSizeList as first argument, got {wrong_type:?}"
@@ -181,7 +181,9 @@ fn map_deduplicate_keys(
 
         if key_is_valid && value_is_valid {
             if num_keys_entries != num_values_entries {
-                return exec_err!("map_deduplicate_keys: keys and values lists in the same row must have equal lengths");
+                return exec_err!(
+                    "map_deduplicate_keys: keys and values lists in the same row must have equal lengths"
+                );
             } else if num_keys_entries != 0 {
                 let mut seen_keys = HashSet::new();
 
diff --git a/crates/sail-function/src/scalar/math/rand_poisson.rs b/crates/sail-function/src/scalar/math/rand_poisson.rs
index f6407c076..7047fe6c4 100644
--- a/crates/sail-function/src/scalar/math/rand_poisson.rs
+++ b/crates/sail-function/src/scalar/math/rand_poisson.rs
@@ -2,10 +2,10 @@ use std::sync::Arc;
 
 use datafusion::arrow::array::Int64Array;
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_datafusion_err, exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use rand::rngs::StdRng;
-use rand::{rng, SeedableRng};
+use rand::{SeedableRng, rng};
 use rand_distr::{Distribution, Poisson};
 
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_types_exec_err};
@@ -66,13 +66,13 @@ impl ScalarUDFImpl for RandPoisson {
                 _ => {
                     return exec_err!(
                         "`random_poisson` expects a scalar Float64 for `lambda`, got {scalar}"
-                    )
+                    );
                 }
             },
             _ => {
                 return exec_err!(
                     "`random_poisson` expects a scalar Float64 for `lambda`, got {lambda}"
-                )
+                );
             }
         };
         if args.len() == 1 {
@@ -86,10 +86,10 @@ impl ScalarUDFImpl for RandPoisson {
                     ScalarValue::Int64(Some(value)) => *value as u64,
                     ScalarValue::UInt64(Some(value)) => *value,
                     ScalarValue::Int64(None) | ScalarValue::UInt64(None) | ScalarValue::Null => {
-                        return invoke_no_seed(lambda, number_rows)
+                        return invoke_no_seed(lambda, number_rows);
                     }
                     _ => {
-                        return exec_err!("`random_poisson` expects an integer seed, got {scalar}")
+                        return exec_err!("`random_poisson` expects an integer seed, got {scalar}");
                     }
                 };
                 let poisson = Poisson::new(lambda).map_err(|e| {
diff --git a/crates/sail-function/src/scalar/math/randn.rs b/crates/sail-function/src/scalar/math/randn.rs
index 3adf091ce..3745edcd2 100644
--- a/crates/sail-function/src/scalar/math/randn.rs
+++ b/crates/sail-function/src/scalar/math/randn.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::array::Float64Array;
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use rand::rng;
 use rand_distr::{Distribution, StandardNormal};
@@ -62,7 +62,7 @@ impl ScalarUDFImpl for Randn {
                     ScalarValue::Int64(Some(value)) => *value,
                     ScalarValue::UInt64(Some(value)) => *value as i64,
                     ScalarValue::Int64(None) | ScalarValue::UInt64(None) | ScalarValue::Null => {
-                        return invoke_no_seed(number_rows)
+                        return invoke_no_seed(number_rows);
                     }
                     _ => return exec_err!("`randn` expects an integer seed, got {scalar}"),
                 };
diff --git a/crates/sail-function/src/scalar/math/random.rs b/crates/sail-function/src/scalar/math/random.rs
index 7f81e3669..4d61f7af1 100644
--- a/crates/sail-function/src/scalar/math/random.rs
+++ b/crates/sail-function/src/scalar/math/random.rs
@@ -2,9 +2,9 @@ use std::sync::Arc;
 
 use datafusion::arrow::array::Float64Array;
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
-use rand::{rng, RngExt};
+use rand::{RngExt, rng};
 
 use super::xorshift::SparkXorShiftRandom;
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_types_exec_err};
@@ -62,7 +62,7 @@ impl ScalarUDFImpl for Random {
                     ScalarValue::Int64(Some(value)) => *value,
                     ScalarValue::UInt64(Some(value)) => *value as i64,
                     ScalarValue::Int64(None) | ScalarValue::UInt64(None) | ScalarValue::Null => {
-                        return invoke_no_seed(number_rows)
+                        return invoke_no_seed(number_rows);
                     }
                     _ => return exec_err!("`random` expects an integer seed, got {scalar}"),
                 };
diff --git a/crates/sail-function/src/scalar/math/spark_abs.rs b/crates/sail-function/src/scalar/math/spark_abs.rs
index aefae7fb0..2ab0f9dfc 100644
--- a/crates/sail-function/src/scalar/math/spark_abs.rs
+++ b/crates/sail-function/src/scalar/math/spark_abs.rs
@@ -1,17 +1,17 @@
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    new_null_array, ArrayRef, AsArray, DurationMicrosecondArray, DurationMillisecondArray,
-    DurationNanosecondArray, DurationSecondArray, Int16Array, Int32Array, Int64Array, Int8Array,
-    IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,
+    ArrayRef, AsArray, DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray,
+    DurationSecondArray, Int8Array, Int16Array, Int32Array, Int64Array, IntervalDayTimeArray,
+    IntervalMonthDayNanoArray, IntervalYearMonthArray, new_null_array,
 };
 use datafusion::arrow::datatypes::{
     DataType, DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
-    DurationSecondType, Int16Type, Int32Type, Int64Type, Int8Type, IntervalDayTimeType,
+    DurationSecondType, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTimeType,
     IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, TimeUnit,
 };
 use datafusion::functions::math::expr_fn::abs;
-use datafusion_common::{exec_datafusion_err, internal_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, internal_err};
 use datafusion_expr::interval_arithmetic::Interval;
 use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
 use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
@@ -114,13 +114,13 @@ impl ScalarUDFImpl for SparkAbs {
             return Err(invalid_arg_count_exec_err("abs", (1, 1), args.len()));
         };
         // Skip the kernel pass when every input row is NULL.
-        if let ColumnarValue::Array(array) = arg {
-            if array.null_count() == array.len() {
-                return Ok(ColumnarValue::Array(new_null_array(
-                    &return_dtype,
-                    array.len(),
-                )));
-            }
+        if let ColumnarValue::Array(array) = arg
+            && array.null_count() == array.len()
+        {
+            return Ok(ColumnarValue::Array(new_null_array(
+                &return_dtype,
+                array.len(),
+            )));
         }
         match arg {
             // Signed integer abs: ANSI=true errors on MIN; ANSI=false wraps
@@ -441,14 +441,12 @@ impl ScalarUDFImpl for SparkAbs {
 
     fn simplify(&self, args: Vec, info: &SimplifyContext) -> Result {
         // Idempotence: abs(abs(x)) = abs(x).
-        if args.len() == 1 {
-            if let Expr::ScalarFunction(inner) = &args[0] {
-                if let Some(inner_abs) = inner.func.inner().downcast_ref::() {
-                    if inner_abs.ansi_mode == self.ansi_mode {
-                        return Ok(ExprSimplifyResult::Simplified(args[0].clone()));
-                    }
-                }
-            }
+        if args.len() == 1
+            && let Expr::ScalarFunction(inner) = &args[0]
+            && let Some(inner_abs) = inner.func.inner().downcast_ref::()
+            && inner_abs.ansi_mode == self.ansi_mode
+        {
+            return Ok(ExprSimplifyResult::Simplified(args[0].clone()));
         }
 
         let dt = info.get_data_type(&args[0])?;
diff --git a/crates/sail-function/src/scalar/math/spark_bin.rs b/crates/sail-function/src/scalar/math/spark_bin.rs
index 07a4b6796..cb18074cf 100644
--- a/crates/sail-function/src/scalar/math/spark_bin.rs
+++ b/crates/sail-function/src/scalar/math/spark_bin.rs
@@ -1,12 +1,12 @@
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    as_largestring_array, as_string_array, new_null_array, Array, ArrayRef, AsArray,
-    PrimitiveArray, StringArray, StringBuilder,
+    Array, ArrayRef, AsArray, PrimitiveArray, StringArray, StringBuilder, as_largestring_array,
+    as_string_array, new_null_array,
 };
 use datafusion::arrow::datatypes::{DataType, Float64Type, Int64Type};
 use datafusion_common::cast::as_string_view_array;
-use datafusion_common::{exec_datafusion_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err};
@@ -176,7 +176,7 @@ where
     // exactly once.
     let (lower, upper) = iter.size_hint();
     let cap = upper.unwrap_or(lower);
-    let mut builder = StringBuilder::with_capacity(cap, cap * 8);
+    let mut builder = StringBuilder::with_capacity(cap, 0);
     let mut scratch = String::with_capacity(64);
     for v in iter {
         match v {
@@ -225,9 +225,7 @@ fn bin_int_array_to_string(
     array: &datafusion::arrow::array::PrimitiveArray,
 ) -> ArrayRef {
     let len = array.len();
-    // 8 chars/row average is a heuristic; the buffer grows as needed but
-    // starts close to the typical size so we avoid most reallocs.
-    let mut builder = StringBuilder::with_capacity(len, len * 8);
+    let mut builder = StringBuilder::with_capacity(len, 0);
     let mut scratch = String::with_capacity(64);
     if array.null_count() == 0 {
         for value in array.values().iter().copied() {
@@ -279,7 +277,7 @@ fn double_to_i64(value: f64, ansi: bool) -> Result {
 }
 
 fn bin_float_array(array: &PrimitiveArray, ansi: bool) -> Result {
-    let mut builder = StringBuilder::with_capacity(array.len(), array.len() * 8);
+    let mut builder = StringBuilder::with_capacity(array.len(), 0);
     let mut scratch = String::with_capacity(64);
     for v in array.iter() {
         match v {
diff --git a/crates/sail-function/src/scalar/math/spark_bround.rs b/crates/sail-function/src/scalar/math/spark_bround.rs
index 8b729a131..b472c4de2 100644
--- a/crates/sail-function/src/scalar/math/spark_bround.rs
+++ b/crates/sail-function/src/scalar/math/spark_bround.rs
@@ -1,8 +1,7 @@
 use std::sync::Arc;
 
-use datafusion::arrow::array::{
-    as_primitive_array, Array, Float32Array, Float64Array, Int32Array, Int64Array,
-};
+use datafusion::arrow::array::{Array, as_primitive_array};
+use datafusion::arrow::compute::binary;
 use datafusion::arrow::datatypes::{DataType, Float32Type, Float64Type, Int32Type, Int64Type};
 use datafusion_common::{Result, ScalarValue};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
@@ -76,7 +75,7 @@ impl ScalarUDFImpl for SparkBRound {
                     "spark_bround",
                     (1, 2),
                     args.len(),
-                ))
+                ));
             }
         };
         let [x, d] = args.as_slice() else {
@@ -117,67 +116,39 @@ impl ScalarUDFImpl for SparkBRound {
             (DataType::Float64, DataType::Int32) => {
                 let x = as_primitive_array::(&x_array);
                 let d = as_primitive_array::(&d_array);
-                let result: Float64Array = x
-                    .iter()
-                    .zip(d.iter())
-                    .map(|(x_val, scale)| match (x_val, scale) {
-                        (Some(x), Some(s)) => Some(spark_bround_f64(x, s)),
-                        _ => None,
-                    })
-                    .collect();
+                let result = binary::(x, d, |x, s| {
+                    spark_bround_f64(x, s)
+                })?;
                 Ok(ColumnarValue::Array(Arc::new(result)))
             }
 
             (DataType::Float32, DataType::Int32) => {
                 let x = as_primitive_array::(&x_array);
                 let d = as_primitive_array::(&d_array);
-                let result: Float32Array = x
-                    .iter()
-                    .zip(d.iter())
-                    .map(|(x_val, scale)| match (x_val, scale) {
-                        (Some(x), Some(s)) => {
-                            let pow = 10f32.powi(s);
-                            Some(round_half_to_even_f32(x * pow) / pow)
-                        }
-                        _ => None,
-                    })
-                    .collect();
+                let result = binary::(x, d, |x, s| {
+                    let pow = 10f32.powi(s);
+                    round_half_to_even_f32(x * pow) / pow
+                })?;
                 Ok(ColumnarValue::Array(Arc::new(result)))
             }
 
             (DataType::Int32, DataType::Int32) => {
                 let x = as_primitive_array::(&x_array);
                 let d = as_primitive_array::(&d_array);
-                let result: Int32Array = x
-                    .iter()
-                    .zip(d.iter())
-                    .map(|(x_val, scale)| match (x_val, scale) {
-                        (Some(x), Some(s)) => {
-                            let x_f64 = x as f64;
-                            let pow = 10f64.powi(s);
-                            Some((round_half_to_even_f64(x_f64 * pow) / pow) as i32)
-                        }
-                        _ => None,
-                    })
-                    .collect();
+                let result = binary::(x, d, |x, s| {
+                    let pow = 10f64.powi(s);
+                    (round_half_to_even_f64(x as f64 * pow) / pow) as i32
+                })?;
                 Ok(ColumnarValue::Array(Arc::new(result)))
             }
 
             (DataType::Int64, DataType::Int32) => {
                 let x = as_primitive_array::(&x_array);
                 let d = as_primitive_array::(&d_array);
-                let result: Int64Array = x
-                    .iter()
-                    .zip(d.iter())
-                    .map(|(x_val, scale)| match (x_val, scale) {
-                        (Some(x), Some(s)) => {
-                            let x_f64 = x as f64;
-                            let pow = 10f64.powi(s);
-                            Some((round_half_to_even_f64(x_f64 * pow).round() / pow).round() as i64)
-                        }
-                        _ => None,
-                    })
-                    .collect();
+                let result = binary::(x, d, |x, s| {
+                    let pow = 10f64.powi(s);
+                    (round_half_to_even_f64(x as f64 * pow).round() / pow).round() as i64
+                })?;
                 Ok(ColumnarValue::Array(Arc::new(result)))
             }
 
@@ -304,8 +275,47 @@ fn spark_bround_f64(x: f64, scale: i32) -> f64 {
 
 #[cfg(test)]
 mod tests {
+    use datafusion::arrow::array::{AsArray, Float64Array, Int32Array};
+    use datafusion::arrow::datatypes::Field;
+    use datafusion_common::config::ConfigOptions;
+
     use super::*;
 
+    #[test]
+    fn test_bround_null_propagation() -> Result<()> {
+        // NULL in `x` on some rows, NULL in `d` (scale) on others.
+        let x = Float64Array::from(vec![Some(2.5), None, Some(3.5), Some(1.25)]);
+        let d = Int32Array::from(vec![Some(0), Some(0), None, Some(1)]);
+        let x_field = Arc::new(Field::new("x", DataType::Float64, true));
+        let d_field = Arc::new(Field::new("d", DataType::Int32, true));
+
+        let result = SparkBRound::new().invoke_with_args(ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Array(Arc::new(x)),
+                ColumnarValue::Array(Arc::new(d)),
+            ],
+            arg_fields: vec![x_field, d_field.clone()],
+            number_rows: 4,
+            return_field: Arc::new(Field::new("result", DataType::Float64, true)),
+            config_options: Arc::new(ConfigOptions::default()),
+        })?;
+
+        let output = result.into_array(4)?;
+        let output = output.as_primitive::();
+
+        // Row 0: valid tie-to-even -> 2.0
+        assert!(output.is_valid(0));
+        assert_eq!(output.value(0), 2.0);
+        // Row 1: NULL x -> NULL
+        assert!(output.is_null(1));
+        // Row 2: NULL scale -> NULL
+        assert!(output.is_null(2));
+        // Row 3: valid -> 1.2
+        assert!(output.is_valid(3));
+        assert_eq!(output.value(3), 1.2);
+        Ok(())
+    }
+
     #[test]
     fn test_bround_f64_positive_scale() {
         assert_eq!(spark_bround_f64(1.25, 1), 1.2);
diff --git a/crates/sail-function/src/scalar/math/spark_ceil_floor.rs b/crates/sail-function/src/scalar/math/spark_ceil_floor.rs
index 4858cc1cd..6f9e77429 100644
--- a/crates/sail-function/src/scalar/math/spark_ceil_floor.rs
+++ b/crates/sail-function/src/scalar/math/spark_ceil_floor.rs
@@ -1,17 +1,19 @@
 use std::sync::Arc;
 
-use datafusion::arrow::array::{ArrayRef, ArrowNativeTypeOp, AsArray};
+use datafusion::arrow::array::{ArrayRef, ArrowNativeTypeOp, AsArray, Float32Array, Float64Array};
+use datafusion::arrow::compute::CastOptions;
 use datafusion::arrow::datatypes::{
-    DataType, Decimal128Type, Field, FieldRef, Float32Type, Float64Type, Int16Type, Int32Type,
-    Int64Type, Int8Type, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE,
+    DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DataType, Decimal128Type, Field, FieldRef,
+    Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
 };
+use datafusion::arrow::util::display::FormatOptions;
 use datafusion_common::{Result, ScalarValue};
 use datafusion_expr::interval_arithmetic::Interval;
 use datafusion_expr::preimage::PreimageResult;
 use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
 use datafusion_expr::{
-    expr, ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
-    Volatility,
+    ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+    expr,
 };
 use num::integer::{div_ceil, div_floor};
 use num::traits::CheckedAdd;
@@ -22,23 +24,31 @@ use crate::error::{
 };
 use crate::scalar::math::utils::decimal::round_decimal_base;
 
+fn extract_call_args<'a>(
+    name: &str,
+    args: &'a [ColumnarValue],
+) -> Result<(&'a ColumnarValue, &'a Option)> {
+    match args.len() {
+        1 => Ok((&args[0], &None)),
+        2 => match &args[1] {
+            ColumnarValue::Scalar(ScalarValue::Int32(value)) => Ok((&args[0], value)),
+            other => Err(unsupported_data_type_exec_err(
+                name,
+                "Target scale must be Integer literal",
+                &other.data_type(),
+            )),
+        },
+        n => Err(invalid_arg_count_exec_err(name, (1, 2), n)),
+    }
+}
+
 fn ceil_floor_coerce_types(name: &str, arg_types: &[DataType]) -> Result> {
     if arg_types.len() == 1 {
-        if arg_types[0].is_numeric() {
-            Ok(vec![ceil_floor_coerce_first_arg(name, &arg_types[0])?])
-        } else {
-            Err(unsupported_data_types_exec_err(
-                name,
-                "Numeric Type",
-                arg_types,
-            ))
-        }
+        Ok(vec![ceil_floor_coerce_first_arg(name, &arg_types[0])?])
     } else if arg_types.len() == 2 {
-        if arg_types[0].is_numeric() && arg_types[1].is_integer() {
-            Ok(vec![
-                ceil_floor_coerce_first_arg(name, &arg_types[0])?,
-                DataType::Int32,
-            ])
+        let first = ceil_floor_coerce_first_arg(name, &arg_types[0])?;
+        if arg_types[1].is_integer() {
+            Ok(vec![first, DataType::Int32])
         } else {
             Err(unsupported_data_types_exec_err(
                 name,
@@ -94,10 +104,12 @@ fn ceil_floor_return_type_from_args(name: &str, args: ReturnFieldArgs) -> Result
                     &target_scale.data_type(),
                 )),
             }?;
-            if target_scale < -38 {
+            // Decimal128 can hold 10^37 (38 digits: 1 followed by 37 zeros) but not 10^38.
+            // target_scale = -n produces a result that's a multiple of 10^n, so n ≤ 37.
+            if target_scale < -37 {
                 return Err(generic_exec_err(
                     name,
-                    "Target scale must be greater than -38",
+                    "Target scale must be greater than or equal to -37",
                 ));
             }
             let (precision, scale) = match expr {
@@ -133,36 +145,36 @@ fn ceil_floor_return_type_from_args(name: &str, args: ReturnFieldArgs) -> Result
             ))
         }
     } else {
+        // Defensive: DataFusion may call `return_field_from_args` in paths
+        // that bypass `coerce_types`. Surface a precise arity error.
         Err(invalid_arg_count_exec_err(name, (1, 2), arg_fields.len()))
     }?;
     Ok(Arc::new(Field::new(name.to_string(), return_type, true)))
 }
 
 fn ceil_floor_coerce_first_arg(name: &str, arg_type: &DataType) -> Result {
-    if arg_type.is_numeric() {
-        match arg_type {
-            DataType::UInt8 => Ok(DataType::Int16),
-            DataType::UInt16 => Ok(DataType::Int32),
-            DataType::UInt32 | DataType::UInt64 => Ok(DataType::Int64),
-            DataType::Decimal256(precision, scale) => {
-                if *precision <= DECIMAL128_MAX_PRECISION && *scale <= DECIMAL128_MAX_SCALE {
-                    Ok(DataType::Decimal128(*precision, *scale))
-                } else {
-                    Err(unsupported_data_type_exec_err(
-                        name,
-                        format!("Decimal Type must have precision <= {DECIMAL128_MAX_PRECISION} and scale <= {DECIMAL128_MAX_SCALE}").as_str(),
-                        arg_type,
-                    ))
-                }
+    match arg_type {
+        DataType::Null => Ok(DataType::Float64),
+        DataType::UInt8 => Ok(DataType::Int16),
+        DataType::UInt16 => Ok(DataType::Int32),
+        DataType::UInt32 | DataType::UInt64 => Ok(DataType::Int64),
+        DataType::Decimal256(precision, scale) => {
+            if *precision <= DECIMAL128_MAX_PRECISION && *scale <= DECIMAL128_MAX_SCALE {
+                Ok(DataType::Decimal128(*precision, *scale))
+            } else {
+                Err(unsupported_data_type_exec_err(
+                    name,
+                    format!("Decimal Type must have precision <= {DECIMAL128_MAX_PRECISION} and scale <= {DECIMAL128_MAX_SCALE}").as_str(),
+                    arg_type,
+                ))
             }
-            other => Ok(other.clone()),
         }
-    } else {
-        Err(unsupported_data_type_exec_err(
+        other if other.is_numeric() => Ok(other.clone()),
+        other => Err(unsupported_data_type_exec_err(
             name,
             "First arg must be Numeric Type",
-            arg_type,
-        ))
+            other,
+        )),
     }
 }
 
@@ -199,31 +211,40 @@ fn ceil_floor_simplify(
         _ => {}
     }
     // Idempotence: floor(floor(x)) = floor(x), ceil(ceil(x)) = ceil(x).
-    if let Expr::ScalarFunction(ref func) = arg {
-        if func.func.inner().is::() && func.args.len() == 1 {
-            return Ok(ExprSimplifyResult::Simplified(arg));
-        }
+    if let Expr::ScalarFunction(ref func) = arg
+        && func.func.inner().is::()
+        && func.args.len() == 1
+    {
+        return Ok(ExprSimplifyResult::Simplified(arg));
     }
     Ok(ExprSimplifyResult::Original(vec![arg]))
 }
 
+/// Spark-compatible `ceil` function.
+/// 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct SparkCeil {
     signature: Signature,
+    ansi_mode: bool,
 }
 
 impl Default for SparkCeil {
     fn default() -> Self {
-        Self::new()
+        Self::new(false)
     }
 }
 
 impl SparkCeil {
-    pub fn new() -> Self {
+    pub fn new(ansi_mode: bool) -> Self {
         Self {
             signature: Signature::user_defined(Volatility::Immutable),
+            ansi_mode,
         }
     }
+
+    pub fn ansi_mode(&self) -> bool {
+        self.ansi_mode
+    }
 }
 
 impl ScalarUDFImpl for SparkCeil {
@@ -251,25 +272,9 @@ impl ScalarUDFImpl for SparkCeil {
     }
 
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
-        let arg_len = args.args.len();
-        let target_scale = if arg_len == 1 {
-            Ok(&None)
-        } else if arg_len == 2 {
-            let target_scale = &args.args[1];
-            match target_scale {
-                ColumnarValue::Scalar(ScalarValue::Int32(value)) => Ok(value),
-                _ => Err(unsupported_data_type_exec_err(
-                    "ceil",
-                    "Target scale must be Integer literal",
-                    &target_scale.data_type(),
-                )),
-            }
-        } else {
-            Err(invalid_arg_count_exec_err("ceil", (1, 2), arg_len))
-        }?;
-        let arg = &args.args[0];
+        let (arg, target_scale) = extract_call_args("ceil", &args.args)?;
         let return_type = args.return_field.data_type();
-        spark_ceil_floor("ceil", arg, target_scale, return_type)
+        spark_ceil_floor("ceil", arg, target_scale, return_type, self.ansi_mode)
     }
 
     fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
@@ -277,23 +282,31 @@ impl ScalarUDFImpl for SparkCeil {
     }
 }
 
+/// Spark-compatible `floor` function.
+/// 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct SparkFloor {
     signature: Signature,
+    ansi_mode: bool,
 }
 
 impl Default for SparkFloor {
     fn default() -> Self {
-        Self::new()
+        Self::new(false)
     }
 }
 
 impl SparkFloor {
-    pub fn new() -> Self {
+    pub fn new(ansi_mode: bool) -> Self {
         Self {
             signature: Signature::user_defined(Volatility::Immutable),
+            ansi_mode,
         }
     }
+
+    pub fn ansi_mode(&self) -> bool {
+        self.ansi_mode
+    }
 }
 
 impl ScalarUDFImpl for SparkFloor {
@@ -321,25 +334,9 @@ impl ScalarUDFImpl for SparkFloor {
     }
 
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
-        let arg_len = args.args.len();
-        let target_scale = if arg_len == 1 {
-            Ok(&None)
-        } else if arg_len == 2 {
-            let target_scale = &args.args[1];
-            match target_scale {
-                ColumnarValue::Scalar(ScalarValue::Int32(value)) => Ok(value),
-                _ => Err(unsupported_data_type_exec_err(
-                    "floor",
-                    "Target scale must be Integer literal",
-                    &target_scale.data_type(),
-                )),
-            }
-        } else {
-            Err(invalid_arg_count_exec_err("floor", (1, 2), arg_len))
-        }?;
-        let arg = &args.args[0];
+        let (arg, target_scale) = extract_call_args("floor", &args.args)?;
         let return_type = args.return_field.data_type();
-        spark_ceil_floor("floor", arg, target_scale, return_type)
+        spark_ceil_floor("floor", arg, target_scale, return_type, self.ansi_mode)
     }
 
     fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
@@ -356,12 +353,28 @@ impl ScalarUDFImpl for SparkFloor {
     }
 }
 
+/// Safe cast options: overflow becomes NULL instead of erroring. Used in
+/// the Float→Decimal path when `spark.sql.ansi.enabled = false`.
+const SAFE_CAST_OPTIONS: CastOptions<'static> = CastOptions {
+    safe: true,
+    format_options: FormatOptions::new(),
+};
+
 fn spark_ceil_floor(
     name: &str,
     arg: &ColumnarValue,
     target_scale: &Option,
     return_type: &DataType,
+    ansi_mode: bool,
 ) -> Result {
+    // Float→Decimal cast options depend on ANSI mode:
+    //   ANSI=true  → None (default: safe=false) → overflow errors, matching Spark ANSI.
+    //   ANSI=false → Some(&SAFE_CAST_OPTIONS) → overflow becomes NULL, matching Spark non-ANSI.
+    let float_cast_options: Option<&CastOptions> = if ansi_mode {
+        None
+    } else {
+        Some(&SAFE_CAST_OPTIONS)
+    };
     if matches!(
         arg.data_type(),
         DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64
@@ -525,9 +538,10 @@ fn spark_ceil_floor(
                 if let Some(target_scale) = *target_scale {
                     let (return_type_precision, return_type_scale) =
                         get_return_type_precision_scale(return_type)?;
-                    let arg = arg.cast_to(
+                    let masked = mask_non_finite_f32(arg);
+                    let arg = masked.cast_to(
                         &DataType::Decimal128(return_type_precision, return_type_scale),
-                        None,
+                        float_cast_options,
                     )?;
                     decimal128_ceil_floor(name, &arg, &Some(target_scale), return_type)
                 } else {
@@ -560,9 +574,10 @@ fn spark_ceil_floor(
                 if let Some(target_scale) = *target_scale {
                     let (return_type_precision, return_type_scale) =
                         get_return_type_precision_scale(return_type)?;
-                    let arg = arg.cast_to(
+                    let masked = mask_non_finite_f64(arg);
+                    let arg = masked.cast_to(
                         &DataType::Decimal128(return_type_precision, return_type_scale),
-                        None,
+                        float_cast_options,
                     )?;
                     decimal128_ceil_floor(name, &arg, &Some(target_scale), return_type)
                 } else {
@@ -585,7 +600,7 @@ fn spark_ceil_floor(
                             as ArrayRef)),
                         _ => Err(unsupported_data_type_exec_err(
                             name,
-                            format!("{}", DataType::Float32).as_str(),
+                            format!("{}", DataType::Float64).as_str(),
                             &arg.data_type(),
                         )),
                     }
@@ -662,7 +677,50 @@ fn decimal128_ceil_floor(
     }
 }
 
-#[inline]
+/// Replace NaN and +/-Infinity values with NULL so they survive the cast to Decimal128.
+/// Spark semantics: `ceil(NaN, s)` and `ceil(+/-Infinity, s)` return NULL even under ANSI,
+/// because NaN/Inf have no valid decimal representation. Finite overflow is intentionally
+/// left to raise a cast error (matching Spark ANSI=true).
+fn mask_non_finite_f32(arg: &ColumnarValue) -> ColumnarValue {
+    match arg {
+        ColumnarValue::Scalar(ScalarValue::Float32(Some(v))) if !v.is_finite() => {
+            ColumnarValue::Scalar(ScalarValue::Float32(None))
+        }
+        ColumnarValue::Array(array) => {
+            let float_arr = array.as_primitive::();
+            let masked: Float32Array = float_arr
+                .iter()
+                .map(|v| match v {
+                    Some(x) if !x.is_finite() => None,
+                    other => other,
+                })
+                .collect();
+            ColumnarValue::Array(Arc::new(masked) as ArrayRef)
+        }
+        other => other.clone(),
+    }
+}
+
+fn mask_non_finite_f64(arg: &ColumnarValue) -> ColumnarValue {
+    match arg {
+        ColumnarValue::Scalar(ScalarValue::Float64(Some(v))) if !v.is_finite() => {
+            ColumnarValue::Scalar(ScalarValue::Float64(None))
+        }
+        ColumnarValue::Array(array) => {
+            let float_arr = array.as_primitive::();
+            let masked: Float64Array = float_arr
+                .iter()
+                .map(|v| match v {
+                    Some(x) if !x.is_finite() => None,
+                    other => other,
+                })
+                .collect();
+            ColumnarValue::Array(Arc::new(masked) as ArrayRef)
+        }
+        other => other.clone(),
+    }
+}
+
 fn ceil_floor_with_target_scale(name: &str, decimal: i128, scale: i8, target_scale: i32) -> i128 {
     // Round to powers of 10 to the left of decimal point when target_scale < 0
     if target_scale < 0 {
@@ -688,7 +746,10 @@ fn ceil_floor_with_target_scale(name: &str, decimal: i128, scale: i8, target_sca
     } else {
         let scale_diff = target_scale - (scale as i32);
         if scale_diff >= 0 {
-            decimal * 10_i128.pow_wrapping(scale_diff as u32)
+            // target_scale >= input scale: no rounding needed. `round_decimal_base`
+            // keeps the return scale equal to the input scale, so the stored value
+            // is unchanged.
+            decimal
         } else {
             let abs_diff = (-scale_diff) as u32;
             if matches!(name, "ceil") {
@@ -790,11 +851,7 @@ fn lit_as_integer(v: &ScalarValue) -> Option {
         ScalarValue::Decimal128(Some(n), _, 0) => Some(*n),
         ScalarValue::Decimal128(Some(n), _, scale) if *scale > 0 => {
             let pow = 10_i128.checked_pow(*scale as u32)?;
-            if n % pow == 0 {
-                Some(n / pow)
-            } else {
-                None
-            }
+            if n % pow == 0 { Some(n / pow) } else { None }
         }
         _ => None,
     }
diff --git a/crates/sail-function/src/scalar/math/spark_conv.rs b/crates/sail-function/src/scalar/math/spark_conv.rs
index f8ea8eb11..9c73de905 100644
--- a/crates/sail-function/src/scalar/math/spark_conv.rs
+++ b/crates/sail-function/src/scalar/math/spark_conv.rs
@@ -1,9 +1,9 @@
 use std::sync::Arc;
 
-use datafusion::arrow::array::{as_primitive_array, ArrayRef, StringArray};
+use datafusion::arrow::array::{ArrayRef, StringArray, as_primitive_array};
 use datafusion::arrow::datatypes::{DataType, Int32Type};
 use datafusion_common::cast::as_generic_string_array;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::error::{
@@ -65,7 +65,7 @@ impl ScalarUDFImpl for SparkConv {
                     "spark_conv",
                     "Scalar String or Int",
                     &num.data_type(),
-                ))
+                ));
             }
         };
 
diff --git a/crates/sail-function/src/scalar/math/spark_div.rs b/crates/sail-function/src/scalar/math/spark_div.rs
index 987953883..d071c7088 100644
--- a/crates/sail-function/src/scalar/math/spark_div.rs
+++ b/crates/sail-function/src/scalar/math/spark_div.rs
@@ -145,7 +145,7 @@ fn interval_div_inner(args: &[ArrayRef]) -> Result {
             return Err(generic_exec_err(
                 "spark_interval_div",
                 "unsupported interval types for division",
-            ))
+            ));
         }
     };
 
diff --git a/crates/sail-function/src/scalar/math/spark_negative.rs b/crates/sail-function/src/scalar/math/spark_negative.rs
index e4102886b..90a0581c0 100644
--- a/crates/sail-function/src/scalar/math/spark_negative.rs
+++ b/crates/sail-function/src/scalar/math/spark_negative.rs
@@ -1,8 +1,8 @@
 use std::sync::{Arc, LazyLock};
 
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::config::ConfigOptions;
 use datafusion_common::Result;
+use datafusion_common::config::ConfigOptions;
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature};
 use datafusion_spark::function::math::negative::SparkNegative as DataFusionNegative;
 
diff --git a/crates/sail-function/src/scalar/math/spark_pmod.rs b/crates/sail-function/src/scalar/math/spark_pmod.rs
index f6f81ce65..c07aaf24b 100644
--- a/crates/sail-function/src/scalar/math/spark_pmod.rs
+++ b/crates/sail-function/src/scalar/math/spark_pmod.rs
@@ -1,7 +1,7 @@
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::Result;
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature};
-use datafusion_spark::function::math::modulus::{spark_pmod, SparkPmod as DataFusionPmod};
+use datafusion_spark::function::math::modulus::{SparkPmod as DataFusionPmod, spark_pmod};
 
 /// Spark `pmod(a, b)` (positive modulo) that honors `spark.sql.ansi.enabled`.
 ///
diff --git a/crates/sail-function/src/scalar/math/spark_signum.rs b/crates/sail-function/src/scalar/math/spark_signum.rs
index 03ed6026d..9e6821030 100644
--- a/crates/sail-function/src/scalar/math/spark_signum.rs
+++ b/crates/sail-function/src/scalar/math/spark_signum.rs
@@ -3,11 +3,11 @@ use std::sync::Arc;
 use datafusion::arrow::array::{ArrayRef, AsArray, Float64Array};
 use datafusion::arrow::datatypes::{
     DataType, Decimal128Type, DurationMicrosecondType, DurationMillisecondType,
-    DurationNanosecondType, DurationSecondType, Float16Type, Float32Type, Float64Type, Int16Type,
-    Int32Type, Int64Type, Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit,
-    IntervalYearMonthType, TimeUnit, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
+    DurationNanosecondType, DurationSecondType, Float16Type, Float32Type, Float64Type, Int8Type,
+    Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit,
+    IntervalYearMonthType, TimeUnit, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
 };
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{
     ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,
 };
@@ -104,38 +104,22 @@ impl ScalarUDFImpl for SparkSignum {
             }
             ColumnarValue::Scalar(ScalarValue::Int8(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_i8 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
+                    if x == 0_i8 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
             ColumnarValue::Scalar(ScalarValue::Int16(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_i16 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
+                    if x == 0_i16 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
             ColumnarValue::Scalar(ScalarValue::Int32(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_i32 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
+                    if x == 0_i32 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
             ColumnarValue::Scalar(ScalarValue::Int64(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_i64 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
+                    if x == 0_i64 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
             ColumnarValue::Scalar(ScalarValue::Float16(val)) => {
@@ -149,22 +133,12 @@ impl ScalarUDFImpl for SparkSignum {
             }
             ColumnarValue::Scalar(ScalarValue::Float32(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_f32 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
-                }))))
-            }
-            ColumnarValue::Scalar(ScalarValue::Float64(val)) => {
-                Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_f64 {
-                        0_f64
-                    } else {
-                        x.signum()
-                    }
+                    if x == 0_f32 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
+            ColumnarValue::Scalar(ScalarValue::Float64(val)) => Ok(ColumnarValue::Scalar(
+                ScalarValue::Float64(val.map(|x| if x == 0_f64 { 0_f64 } else { x.signum() })),
+            )),
             ColumnarValue::Scalar(ScalarValue::Decimal128(val, _precision, _scale)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
                     if x == 0 {
@@ -178,11 +152,7 @@ impl ScalarUDFImpl for SparkSignum {
             }
             ColumnarValue::Scalar(ScalarValue::IntervalYearMonth(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_i32 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
+                    if x == 0_i32 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
             ColumnarValue::Scalar(ScalarValue::IntervalDayTime(val)) => {
@@ -223,11 +193,7 @@ impl ScalarUDFImpl for SparkSignum {
             | ColumnarValue::Scalar(ScalarValue::DurationMicrosecond(val))
             | ColumnarValue::Scalar(ScalarValue::DurationNanosecond(val)) => {
                 Ok(ColumnarValue::Scalar(ScalarValue::Float64(val.map(|x| {
-                    if x == 0_i64 {
-                        0_f64
-                    } else {
-                        x.signum() as f64
-                    }
+                    if x == 0_i64 { 0_f64 } else { x.signum() as f64 }
                 }))))
             }
             ColumnarValue::Array(array) => {
@@ -273,43 +239,27 @@ impl ScalarUDFImpl for SparkSignum {
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Int8 => {
-                        let result: Float64Array = array.as_primitive::().unary(|x| {
-                            if x == 0_i8 {
-                                0_f64
-                            } else {
-                                x.signum() as f64
-                            }
-                        });
+                        let result: Float64Array = array
+                            .as_primitive::()
+                            .unary(|x| if x == 0_i8 { 0_f64 } else { x.signum() as f64 });
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Int16 => {
-                        let result: Float64Array = array.as_primitive::().unary(|x| {
-                            if x == 0_i16 {
-                                0_f64
-                            } else {
-                                x.signum() as f64
-                            }
-                        });
+                        let result: Float64Array = array
+                            .as_primitive::()
+                            .unary(|x| if x == 0_i16 { 0_f64 } else { x.signum() as f64 });
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Int32 => {
-                        let result: Float64Array = array.as_primitive::().unary(|x| {
-                            if x == 0_i32 {
-                                0_f64
-                            } else {
-                                x.signum() as f64
-                            }
-                        });
+                        let result: Float64Array = array
+                            .as_primitive::()
+                            .unary(|x| if x == 0_i32 { 0_f64 } else { x.signum() as f64 });
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Int64 => {
-                        let result: Float64Array = array.as_primitive::().unary(|x| {
-                            if x == 0_i64 {
-                                0_f64
-                            } else {
-                                x.signum() as f64
-                            }
-                        });
+                        let result: Float64Array = array
+                            .as_primitive::()
+                            .unary(|x| if x == 0_i64 { 0_f64 } else { x.signum() as f64 });
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Float16 => {
@@ -323,23 +273,15 @@ impl ScalarUDFImpl for SparkSignum {
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Float32 => {
-                        let result: Float64Array = array.as_primitive::().unary(|x| {
-                            if x == 0_f32 {
-                                0_f64
-                            } else {
-                                x.signum() as f64
-                            }
-                        });
+                        let result: Float64Array = array
+                            .as_primitive::()
+                            .unary(|x| if x == 0_f32 { 0_f64 } else { x.signum() as f64 });
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Float64 => {
-                        let result: Float64Array = array.as_primitive::().unary(|x| {
-                            if x == 0_f64 {
-                                0_f64
-                            } else {
-                                x.signum()
-                            }
-                        });
+                        let result: Float64Array = array
+                            .as_primitive::()
+                            .unary(|x| if x == 0_f64 { 0_f64 } else { x.signum() });
                         Ok(Arc::new(result) as ArrayRef)
                     }
                     DataType::Decimal128(_p, _s) => {
diff --git a/crates/sail-function/src/scalar/math/spark_try_add.rs b/crates/sail-function/src/scalar/math/spark_try_add.rs
index 76fe677b0..c2faefcfc 100644
--- a/crates/sail-function/src/scalar/math/spark_try_add.rs
+++ b/crates/sail-function/src/scalar/math/spark_try_add.rs
@@ -57,12 +57,14 @@ impl ScalarUDFImpl for SparkTryAdd {
             | [DataType::Int32, DataType::Interval(MonthDayNano)]
             | [DataType::Interval(MonthDayNano), DataType::Int64]
             | [DataType::Int64, DataType::Interval(MonthDayNano)]
-            | [DataType::Interval(MonthDayNano), DataType::Interval(MonthDayNano)] => {
-                Ok(DataType::Interval(MonthDayNano))
-            }
-            [DataType::Timestamp(Microsecond, _), DataType::Duration(Microsecond)] => {
-                Ok(DataType::Timestamp(Microsecond, None))
-            }
+            | [
+                DataType::Interval(MonthDayNano),
+                DataType::Interval(MonthDayNano),
+            ] => Ok(DataType::Interval(MonthDayNano)),
+            [
+                DataType::Timestamp(Microsecond, _),
+                DataType::Duration(Microsecond),
+            ] => Ok(DataType::Timestamp(Microsecond, None)),
 
             _ => Err(unsupported_data_types_exec_err(
                 "try_add",
diff --git a/crates/sail-function/src/scalar/math/spark_try_mult.rs b/crates/sail-function/src/scalar/math/spark_try_mult.rs
index bc4aacc8e..448b3d6e2 100644
--- a/crates/sail-function/src/scalar/math/spark_try_mult.rs
+++ b/crates/sail-function/src/scalar/math/spark_try_mult.rs
@@ -1,13 +1,13 @@
 use std::sync::Arc;
 
 use datafusion::arrow::array::{Array, ArrayRef, AsArray};
-use datafusion::arrow::compute::{cast_with_options, CastOptions};
+use datafusion::arrow::compute::{CastOptions, cast_with_options};
 use datafusion::arrow::datatypes::IntervalUnit::{MonthDayNano, YearMonth};
 use datafusion::arrow::datatypes::{
     DataType, Int32Type, Int64Type, IntervalMonthDayNanoType, IntervalYearMonthType,
 };
-use datafusion_common::utils::take_function_args;
 use datafusion_common::Result;
+use datafusion_common::utils::take_function_args;
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use datafusion_functions::utils::make_scalar_function;
 
@@ -51,14 +51,22 @@ impl ScalarUDFImpl for SparkTryMult {
             [DataType::Int64, DataType::Int64]
             | [DataType::Int32, DataType::Int64]
             | [DataType::Int64, DataType::Int32] => Ok(DataType::Int64),
-            [DataType::Interval(YearMonth), DataType::Int32 | DataType::Int64]
-            | [DataType::Int32 | DataType::Int64, DataType::Interval(YearMonth)] => {
-                Ok(DataType::Interval(YearMonth))
-            }
-            [DataType::Interval(MonthDayNano), DataType::Int32 | DataType::Int64]
-            | [DataType::Int32 | DataType::Int64, DataType::Interval(MonthDayNano)] => {
-                Ok(DataType::Interval(MonthDayNano))
-            }
+            [
+                DataType::Interval(YearMonth),
+                DataType::Int32 | DataType::Int64,
+            ]
+            | [
+                DataType::Int32 | DataType::Int64,
+                DataType::Interval(YearMonth),
+            ] => Ok(DataType::Interval(YearMonth)),
+            [
+                DataType::Interval(MonthDayNano),
+                DataType::Int32 | DataType::Int64,
+            ]
+            | [
+                DataType::Int32 | DataType::Int64,
+                DataType::Interval(MonthDayNano),
+            ] => Ok(DataType::Interval(MonthDayNano)),
 
             _ => Err(unsupported_data_types_exec_err(
                 "try_multiply",
diff --git a/crates/sail-function/src/scalar/math/spark_try_subtract.rs b/crates/sail-function/src/scalar/math/spark_try_subtract.rs
index 2490f5b90..07e8df689 100644
--- a/crates/sail-function/src/scalar/math/spark_try_subtract.rs
+++ b/crates/sail-function/src/scalar/math/spark_try_subtract.rs
@@ -48,9 +48,9 @@ impl ScalarUDFImpl for SparkTrySubtract {
 
     fn return_type(&self, arg_types: &[DataType]) -> Result {
         match arg_types {
-            [DataType::Int32, DataType::Int32]=> Ok(DataType::Int32),
+            [DataType::Int32, DataType::Int32] => Ok(DataType::Int32),
             [DataType::Int64, DataType::Int64]
-            | [DataType::Int32,DataType::Int64]
+            | [DataType::Int32, DataType::Int64]
             | [DataType::Int64, DataType::Int32] => Ok(DataType::Int64),
             [DataType::Date32, DataType::Int32]
             | [DataType::Date32, DataType::Interval(YearMonth)]
@@ -58,11 +58,17 @@ impl ScalarUDFImpl for SparkTrySubtract {
             | [DataType::Int32, DataType::Date32]
             | [DataType::Interval(YearMonth), DataType::Date32]
             | [DataType::Interval(MonthDayNano), DataType::Date32] => Ok(DataType::Date32),
-            [DataType::Interval(YearMonth), DataType::Interval(YearMonth)] => Ok(DataType::Interval(YearMonth)),
-            [DataType::Interval(MonthDayNano), DataType::Interval(MonthDayNano)] => Ok(DataType::Interval(MonthDayNano)),
-            [DataType::Timestamp(Microsecond, _), DataType::Duration(Microsecond)] => {
-                Ok(DataType::Timestamp(Microsecond, None))
+            [DataType::Interval(YearMonth), DataType::Interval(YearMonth)] => {
+                Ok(DataType::Interval(YearMonth))
             }
+            [
+                DataType::Interval(MonthDayNano),
+                DataType::Interval(MonthDayNano),
+            ] => Ok(DataType::Interval(MonthDayNano)),
+            [
+                DataType::Timestamp(Microsecond, _),
+                DataType::Duration(Microsecond),
+            ] => Ok(DataType::Timestamp(Microsecond, None)),
 
             _ => Err(unsupported_data_types_exec_err(
                 "try_subtract",
@@ -102,13 +108,13 @@ impl ScalarUDFImpl for SparkTrySubtract {
         match (left_arr.data_type(), right_arr.data_type()) {
             (DataType::Int32, DataType::Int32) => {
                 let l = left_arr.as_primitive::();
-                let r= right_arr.as_primitive::();
+                let r = right_arr.as_primitive::();
                 let result = try_binary_op_primitive::(l, r, i32::checked_sub);
 
                 binary_op_scalar_or_array(left, right, result)
             }
             (DataType::Int64, DataType::Int64) => {
-                let l= left_arr.as_primitive::();
+                let l = left_arr.as_primitive::();
                 let r = right_arr.as_primitive::();
                 let result = try_binary_op_primitive::(l, r, i64::checked_sub);
 
@@ -116,33 +122,30 @@ impl ScalarUDFImpl for SparkTrySubtract {
             }
             (DataType::Date32, DataType::Int32) => {
                 let l = left_arr.as_primitive::();
-                let r= right_arr.as_primitive::();
-                let result= try_binary_op_date32_i32(l, r, i32::checked_sub);
+                let r = right_arr.as_primitive::();
+                let result = try_binary_op_date32_i32(l, r, i32::checked_sub);
 
                 binary_op_scalar_or_array(left, right, result)
             }
             (DataType::Date32, DataType::Interval(YearMonth)) => {
                 let l = left_arr.as_primitive::();
                 let r = right_arr.as_primitive::();
-                let result: Date32Array = try_op_date32_interval_yearmonth(l, r, |d, m| add_months(d, -m));
+                let result: Date32Array =
+                    try_op_date32_interval_yearmonth(l, r, |d, m| add_months(d, -m));
 
                 binary_op_scalar_or_array(left, right, result)
             }
             (DataType::Date32, DataType::Interval(MonthDayNano)) => {
-                let l= left_arr.as_primitive::();
+                let l = left_arr.as_primitive::();
                 let r = right_arr.as_primitive::();
                 let result = try_op_date32_monthdaynano(l, r, |x| {
-                    IntervalMonthDayNano::new(
-                        -x.months,
-                        -x.days,
-                        -x.nanoseconds,
-                    )
+                    IntervalMonthDayNano::new(-x.months, -x.days, -x.nanoseconds)
                 });
 
                 binary_op_scalar_or_array(left, right, result)
             }
             (DataType::Interval(YearMonth), DataType::Interval(YearMonth)) => {
-                let l= left_arr.as_primitive::();
+                let l = left_arr.as_primitive::();
                 let r = right_arr.as_primitive::();
                 let result = try_op_interval_yearmonth(l, r, i32::checked_sub);
 
@@ -152,7 +155,7 @@ impl ScalarUDFImpl for SparkTrySubtract {
                 let l = left_arr.as_primitive::();
                 let r = right_arr.as_primitive::();
                 let negated_r = r.iter().map(|opt| opt.map(|v| v.neg())).collect();
-                let result  = try_add_interval_monthdaynano(l, &negated_r);
+                let result = try_add_interval_monthdaynano(l, &negated_r);
 
                 binary_op_scalar_or_array(left, right, result)
             }
@@ -163,7 +166,6 @@ impl ScalarUDFImpl for SparkTrySubtract {
                 let r = right_arr.as_primitive::();
                 let result = try_op_timestamp_duration(l, r, i64::checked_sub);
 
-
                 if matches!(left, ColumnarValue::Scalar(_))
                     && matches!(right, ColumnarValue::Scalar(_))
                 {
diff --git a/crates/sail-function/src/scalar/math/spark_unhex.rs b/crates/sail-function/src/scalar/math/spark_unhex.rs
index 3bb039cb6..dd99c4980 100644
--- a/crates/sail-function/src/scalar/math/spark_unhex.rs
+++ b/crates/sail-function/src/scalar/math/spark_unhex.rs
@@ -2,8 +2,8 @@ use std::fmt::Write;
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    as_dictionary_array, as_largestring_array, as_string_array, BinaryBuilder, OffsetSizeTrait,
-    StringArray,
+    BinaryBuilder, OffsetSizeTrait, StringArray, as_dictionary_array, as_largestring_array,
+    as_string_array,
 };
 use datafusion::arrow::datatypes::{DataType, Int32Type};
 use datafusion::logical_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility};
@@ -11,7 +11,7 @@ use datafusion_common::cast::{
     as_binary_array, as_fixed_size_binary_array, as_generic_string_array, as_int64_array,
     as_string_view_array,
 };
-use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue};
+use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err};
 use datafusion_expr::ScalarFunctionArgs;
 use datafusion_expr_common::signature::TypeSignature;
 
diff --git a/crates/sail-function/src/scalar/math/spark_uniform.rs b/crates/sail-function/src/scalar/math/spark_uniform.rs
index 75bf96c22..8bcd164e0 100644
--- a/crates/sail-function/src/scalar/math/spark_uniform.rs
+++ b/crates/sail-function/src/scalar/math/spark_uniform.rs
@@ -1,17 +1,17 @@
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    new_null_array, Array, ArrayRef, Decimal128Array, Float32Array, Float64Array, Int16Array,
-    Int32Array, Int64Array, Int8Array,
+    Array, ArrayRef, Decimal128Array, Float32Array, Float64Array, Int8Array, Int16Array,
+    Int32Array, Int64Array, new_null_array,
 };
 use datafusion::arrow::datatypes::{
-    DataType, Field, FieldRef, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE,
+    DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DataType, Field, FieldRef,
 };
-use datafusion_common::{internal_err, Result};
+use datafusion_common::{Result, internal_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
-use rand::{rng, RngExt};
+use rand::{RngExt, rng};
 
 use super::xorshift::SparkXorShiftRandom;
 use crate::error::{
@@ -251,7 +251,7 @@ impl ScalarUDFImpl for SparkUniform {
                         "uniform",
                         "INT or BIGINT type for seed",
                         &arg_types[2..],
-                    ))
+                    ));
                 }
             };
             coerced_types.push(seed_type);
diff --git a/crates/sail-function/src/scalar/misc/hll_sketch.rs b/crates/sail-function/src/scalar/misc/hll_sketch.rs
index bcfbb1273..e5f3ab298 100644
--- a/crates/sail-function/src/scalar/misc/hll_sketch.rs
+++ b/crates/sail-function/src/scalar/misc/hll_sketch.rs
@@ -1,10 +1,10 @@
 use std::sync::Arc;
 
 use arrow::array::{
-    new_null_array, Array, ArrayRef, BinaryArray, BinaryBuilder, BooleanArray, Int64Builder,
+    Array, ArrayRef, BinaryArray, BinaryBuilder, BooleanArray, Int64Builder, new_null_array,
 };
 use arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, TypeSignature, Volatility};
diff --git a/crates/sail-function/src/scalar/misc/monotonically_increasing_id.rs b/crates/sail-function/src/scalar/misc/monotonically_increasing_id.rs
index 4729ec42f..1bd914562 100644
--- a/crates/sail-function/src/scalar/misc/monotonically_increasing_id.rs
+++ b/crates/sail-function/src/scalar/misc/monotonically_increasing_id.rs
@@ -1,5 +1,5 @@
 use datafusion::arrow::datatypes::DataType;
-use datafusion::common::{exec_err, Result};
+use datafusion::common::{Result, exec_err};
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, Volatility};
diff --git a/crates/sail-function/src/scalar/misc/raise_error.rs b/crates/sail-function/src/scalar/misc/raise_error.rs
index 0eb0d29fb..a88cdb45c 100644
--- a/crates/sail-function/src/scalar/misc/raise_error.rs
+++ b/crates/sail-function/src/scalar/misc/raise_error.rs
@@ -3,7 +3,7 @@ use datafusion::arrow::datatypes::DataType;
 use datafusion::common::{DataFusionError, Result};
 use datafusion::logical_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility};
 use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array};
-use datafusion_common::{internal_err, ScalarValue};
+use datafusion_common::{ScalarValue, internal_err};
 use datafusion_expr::ScalarFunctionArgs;
 use sail_common_datafusion::utils::items::ItemTaker;
 
diff --git a/crates/sail-function/src/scalar/misc/spark_aes.rs b/crates/sail-function/src/scalar/misc/spark_aes.rs
index 5ded52f16..dfe9f26d7 100644
--- a/crates/sail-function/src/scalar/misc/spark_aes.rs
+++ b/crates/sail-function/src/scalar/misc/spark_aes.rs
@@ -13,7 +13,7 @@ use datafusion::arrow::array::{
     StringArray, StringViewArray,
 };
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError, Result, ScalarValue};
+use datafusion_common::{DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 pub type Aes192Gcm = AesGcm;
@@ -151,41 +151,53 @@ impl ScalarUDFImpl for SparkAESEncrypt {
                 }
                 match array.data_type() {
                     DataType::Binary => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to BinaryArray"))?;
+                        let array = array.as_any().downcast_ref::().ok_or_else(
+                            || {
+                                exec_datafusion_err!(
+                                    "Spark `aes_encrypt`: Failed to downcast Expr to BinaryArray"
+                                )
+                            },
+                        )?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::BinaryView => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::FixedSizeBinary(_) => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to FixedSizeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::LargeBinary => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::Utf8 => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to StringArray"))?;
+                        let array = array.as_any().downcast_ref::().ok_or_else(
+                            || {
+                                exec_datafusion_err!(
+                                    "Spark `aes_encrypt`: Failed to downcast Expr to StringArray"
+                                )
+                            },
+                        )?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::LargeUtf8 => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to LargeStringArray"))?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::Utf8View => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Expr to StringViewArray"))?;
                         Ok(array.value(0).as_bytes())
-                    },
-                    other => exec_err!("Spark `aes_encrypt`: Expr array must be BINARY or STRING, got array of type {other}")
+                    }
+                    other => exec_err!(
+                        "Spark `aes_encrypt`: Expr array must be BINARY or STRING, got array of type {other}"
+                    ),
                 }
             }
             other => exec_err!("Spark `aes_encrypt`: Expr must be BINARY or STRING, got {other:?}"),
@@ -207,41 +219,63 @@ impl ScalarUDFImpl for SparkAESEncrypt {
                 }
                 match array.data_type() {
                     DataType::Binary => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to BinaryArray"))?;
+                        let array =
+                            array
+                                .as_any()
+                                .downcast_ref::()
+                                .ok_or_else(|| {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_encrypt`: Failed to downcast Key to BinaryArray"
+                                    )
+                                })?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::BinaryView => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::FixedSizeBinary(_) => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to FixedSizeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::LargeBinary => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::Utf8 => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to StringArray"))?;
+                        let array =
+                            array
+                                .as_any()
+                                .downcast_ref::()
+                                .ok_or_else(|| {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_encrypt`: Failed to downcast Key to StringArray"
+                                    )
+                                })?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::LargeUtf8 => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to LargeStringArray"))?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::Utf8View => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to StringViewArray"))?;
+                        let array = array
+                            .as_any()
+                            .downcast_ref::()
+                            .ok_or_else(|| {
+                                exec_datafusion_err!(
+                                    "Spark `aes_encrypt`: Failed to downcast Key to StringViewArray"
+                                )
+                            })?;
                         Ok(array.value(0).as_bytes())
-                    },
-                    other => exec_err!("Spark `aes_encrypt`: Key array must be BINARY or STRING, got array of type {other}")
+                    }
+                    other => exec_err!(
+                        "Spark `aes_encrypt`: Key array must be BINARY or STRING, got array of type {other}"
+                    ),
                 }
             }
             other => exec_err!("Spark `aes_encrypt`: Key must be BINARY, got {other:?}"),
@@ -265,18 +299,20 @@ impl ScalarUDFImpl for SparkAESEncrypt {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Mode to StringArray"))?;
                             encryption_name_to_mode(array.value(0))
-                        },
+                        }
                         DataType::LargeUtf8 => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Mode to LargeStringArray"))?;
                             encryption_name_to_mode(array.value(0))
-                        },
+                        }
                         DataType::Utf8View => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Mode to StringViewArray"))?;
                             encryption_name_to_mode(array.value(0))
-                        },
-                        other => exec_err!("Spark `aes_encrypt`: Mode array must be STRING, got array of type {other}")
+                        }
+                        other => exec_err!(
+                            "Spark `aes_encrypt`: Mode array must be STRING, got array of type {other}"
+                        ),
                     }
                 }
                 other => exec_err!("Spark `aes_encrypt`: Mode must be a STRING, got {other:?}"),
@@ -302,41 +338,53 @@ impl ScalarUDFImpl for SparkAESEncrypt {
                     }
                     match array.data_type() {
                         DataType::Binary => {
-                            let array = array.as_any().downcast_ref::()
-                                .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast IV to BinaryArray"))?;
+                            let array = array.as_any().downcast_ref::().ok_or_else(
+                                || {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_encrypt`: Failed to downcast IV to BinaryArray"
+                                    )
+                                },
+                            )?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::BinaryView => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast IV to LargeBinaryArray"))?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::FixedSizeBinary(_) => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast IV to FixedSizeBinaryArray"))?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::LargeBinary => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast IV to LargeBinaryArray"))?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::Utf8 => {
-                            let array = array.as_any().downcast_ref::()
-                                .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to StringArray"))?;
+                            let array = array.as_any().downcast_ref::().ok_or_else(
+                                || {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_encrypt`: Failed to downcast Key to StringArray"
+                                    )
+                                },
+                            )?;
                             Ok(array.value(0).as_bytes())
-                        },
+                        }
                         DataType::LargeUtf8 => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to LargeStringArray"))?;
                             Ok(array.value(0).as_bytes())
-                        },
+                        }
                         DataType::Utf8View => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast Key to StringViewArray"))?;
                             Ok(array.value(0).as_bytes())
-                        },
-                        other => exec_err!("Spark `aes_encrypt`: IV must be BINARY or STRING, got array of type {other}")
+                        }
+                        other => exec_err!(
+                            "Spark `aes_encrypt`: IV must be BINARY or STRING, got array of type {other}"
+                        ),
                     }
                 }
                 other => {
@@ -395,21 +443,28 @@ impl ScalarUDFImpl for SparkAESEncrypt {
                     }
                     let aad = match array.data_type() {
                         DataType::Utf8 => {
-                            let array = array.as_any().downcast_ref::()
-                                .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast AAD to StringArray"))?;
+                            let array = array.as_any().downcast_ref::().ok_or_else(
+                                || {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_encrypt`: Failed to downcast AAD to StringArray"
+                                    )
+                                },
+                            )?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::LargeUtf8 => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast AAD to LargeStringArray"))?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::Utf8View => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_encrypt`: Failed to downcast AAD to StringViewArray"))?;
                             Ok(array.value(0))
-                        },
-                        other => exec_err!("Spark `aes_encrypt`: AAD array must be STRING, got array of type {other}")
+                        }
+                        other => exec_err!(
+                            "Spark `aes_encrypt`: AAD array must be STRING, got array of type {other}"
+                        ),
                     }?;
                     if aad.is_empty() {
                         // If none is provided, Spark passes up an empty string.
@@ -598,41 +653,53 @@ impl ScalarUDFImpl for SparkAESDecrypt {
                 }
                 match array.data_type() {
                     DataType::Binary => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to BinaryArray"))?;
+                        let array = array.as_any().downcast_ref::().ok_or_else(
+                            || {
+                                exec_datafusion_err!(
+                                    "Spark `aes_decrypt`: Failed to downcast Expr to BinaryArray"
+                                )
+                            },
+                        )?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::BinaryView => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::FixedSizeBinary(_) => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to FixedSizeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::LargeBinary => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::Utf8 => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to StringArray"))?;
+                        let array = array.as_any().downcast_ref::().ok_or_else(
+                            || {
+                                exec_datafusion_err!(
+                                    "Spark `aes_decrypt`: Failed to downcast Expr to StringArray"
+                                )
+                            },
+                        )?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::LargeUtf8 => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to LargeStringArray"))?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::Utf8View => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Expr to StringViewArray"))?;
                         Ok(array.value(0).as_bytes())
-                    },
-                    other => exec_err!("Spark `aes_decrypt`: Expr array must be BINARY or STRING, got array of type {other}")
+                    }
+                    other => exec_err!(
+                        "Spark `aes_decrypt`: Expr array must be BINARY or STRING, got array of type {other}"
+                    ),
                 }
             }
             other => exec_err!("Spark `aes_decrypt`: Expr must be BINARY or STRING, got {other:?}"),
@@ -654,41 +721,63 @@ impl ScalarUDFImpl for SparkAESDecrypt {
                 }
                 match array.data_type() {
                     DataType::Binary => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to BinaryArray"))?;
+                        let array =
+                            array
+                                .as_any()
+                                .downcast_ref::()
+                                .ok_or_else(|| {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_decrypt`: Failed to downcast Key to BinaryArray"
+                                    )
+                                })?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::BinaryView => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::FixedSizeBinary(_) => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to FixedSizeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::LargeBinary => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to LargeBinaryArray"))?;
                         Ok(array.value(0))
-                    },
+                    }
                     DataType::Utf8 => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to StringArray"))?;
+                        let array =
+                            array
+                                .as_any()
+                                .downcast_ref::()
+                                .ok_or_else(|| {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_decrypt`: Failed to downcast Key to StringArray"
+                                    )
+                                })?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::LargeUtf8 => {
                         let array = array.as_any().downcast_ref::()
                             .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to LargeStringArray"))?;
                         Ok(array.value(0).as_bytes())
-                    },
+                    }
                     DataType::Utf8View => {
-                        let array = array.as_any().downcast_ref::()
-                            .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Key to StringViewArray"))?;
+                        let array = array
+                            .as_any()
+                            .downcast_ref::()
+                            .ok_or_else(|| {
+                                exec_datafusion_err!(
+                                    "Spark `aes_decrypt`: Failed to downcast Key to StringViewArray"
+                                )
+                            })?;
                         Ok(array.value(0).as_bytes())
-                    },
-                    other => exec_err!("Spark `aes_decrypt`: Key array must be BINARY or STRING, got array of type {other}")
+                    }
+                    other => exec_err!(
+                        "Spark `aes_decrypt`: Key array must be BINARY or STRING, got array of type {other}"
+                    ),
                 }
             }
             other => exec_err!("Spark `aes_decrypt`: Key must be BINARY, got {other:?}"),
@@ -712,18 +801,20 @@ impl ScalarUDFImpl for SparkAESDecrypt {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Mode to StringArray"))?;
                             encryption_name_to_mode(array.value(0))
-                        },
+                        }
                         DataType::LargeUtf8 => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Mode to LargeStringArray"))?;
                             encryption_name_to_mode(array.value(0))
-                        },
+                        }
                         DataType::Utf8View => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast Mode to StringViewArray"))?;
                             encryption_name_to_mode(array.value(0))
-                        },
-                        other => exec_err!("Spark `aes_decrypt`: Mode array must be STRING, got array of type {other}")
+                        }
+                        other => exec_err!(
+                            "Spark `aes_decrypt`: Mode array must be STRING, got array of type {other}"
+                        ),
                     }
                 }
                 other => exec_err!("Spark `aes_decrypt`: Mode must be a STRING, got {other:?}"),
@@ -752,21 +843,28 @@ impl ScalarUDFImpl for SparkAESDecrypt {
                     }
                     let aad = match array.data_type() {
                         DataType::Utf8 => {
-                            let array = array.as_any().downcast_ref::()
-                                .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast AAD to StringArray"))?;
+                            let array = array.as_any().downcast_ref::().ok_or_else(
+                                || {
+                                    exec_datafusion_err!(
+                                        "Spark `aes_decrypt`: Failed to downcast AAD to StringArray"
+                                    )
+                                },
+                            )?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::LargeUtf8 => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast AAD to LargeStringArray"))?;
                             Ok(array.value(0))
-                        },
+                        }
                         DataType::Utf8View => {
                             let array = array.as_any().downcast_ref::()
                                 .ok_or_else(|| exec_datafusion_err!("Spark `aes_decrypt`: Failed to downcast AAD to StringViewArray"))?;
                             Ok(array.value(0))
-                        },
-                        other => exec_err!("Spark `aes_decrypt`: AAD array must be STRING, got array of type {other}")
+                        }
+                        other => exec_err!(
+                            "Spark `aes_decrypt`: AAD array must be STRING, got array of type {other}"
+                        ),
                     }?;
                     if aad.is_empty() {
                         // If none is provided, Spark passes up an empty string.
diff --git a/crates/sail-function/src/scalar/misc/spark_partition_id.rs b/crates/sail-function/src/scalar/misc/spark_partition_id.rs
index 8d536bf3d..d28ed1c52 100644
--- a/crates/sail-function/src/scalar/misc/spark_partition_id.rs
+++ b/crates/sail-function/src/scalar/misc/spark_partition_id.rs
@@ -1,5 +1,5 @@
 use datafusion::arrow::datatypes::DataType;
-use datafusion::common::{exec_err, Result};
+use datafusion::common::{Result, exec_err};
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, Volatility};
diff --git a/crates/sail-function/src/scalar/misc/theta_sketch.rs b/crates/sail-function/src/scalar/misc/theta_sketch.rs
index 4b2017a8a..f0b12ded6 100644
--- a/crates/sail-function/src/scalar/misc/theta_sketch.rs
+++ b/crates/sail-function/src/scalar/misc/theta_sketch.rs
@@ -1,18 +1,18 @@
 use std::sync::Arc;
 
 use arrow::array::{
-    new_null_array, Array, ArrayRef, BinaryArray, BinaryBuilder, Int32Array, Int64Builder,
+    Array, ArrayRef, BinaryArray, BinaryBuilder, Int32Array, Int64Builder, new_null_array,
 };
 use arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, TypeSignature, Volatility};
 
 use crate::functions_utils::make_scalar_function;
 use crate::theta_sketch::{
-    difference_sketch_bytes, estimate_sketch, intersect_sketch_bytes, union_sketch_bytes,
-    validate_lg_nom_entries, DEFAULT_LG_NOM_ENTRIES,
+    DEFAULT_LG_NOM_ENTRIES, difference_sketch_bytes, estimate_sketch, intersect_sketch_bytes,
+    union_sketch_bytes, validate_lg_nom_entries,
 };
 
 #[derive(Debug, PartialEq, Eq, Hash)]
diff --git a/crates/sail-function/src/scalar/multi_expr.rs b/crates/sail-function/src/scalar/multi_expr.rs
index 791de3e7e..cd86e3d4c 100644
--- a/crates/sail-function/src/scalar/multi_expr.rs
+++ b/crates/sail-function/src/scalar/multi_expr.rs
@@ -1,5 +1,5 @@
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{plan_err, Result};
+use datafusion_common::{Result, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 /// A placeholder UDF used to represent a list of expressions that come from the output
diff --git a/crates/sail-function/src/scalar/predicate/rewrite_like_pattern.rs b/crates/sail-function/src/scalar/predicate/rewrite_like_pattern.rs
index fcd246948..523d84feb 100644
--- a/crates/sail-function/src/scalar/predicate/rewrite_like_pattern.rs
+++ b/crates/sail-function/src/scalar/predicate/rewrite_like_pattern.rs
@@ -4,7 +4,7 @@ use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait};
 use arrow::datatypes::DataType;
 use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
 use datafusion_common::types::logical_string;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{
     Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass,
     Volatility,
diff --git a/crates/sail-function/src/scalar/spark_struct_rename.rs b/crates/sail-function/src/scalar/spark_struct_rename.rs
index 307134517..284393601 100644
--- a/crates/sail-function/src/scalar/spark_struct_rename.rs
+++ b/crates/sail-function/src/scalar/spark_struct_rename.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::array::{Array, ArrayRef, AsArray, FixedSizeListArray, StructArray};
 use datafusion::arrow::datatypes::{DataType, Field, Fields};
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 /// Spark-compatible "rename struct fields by position".
diff --git a/crates/sail-function/src/scalar/spark_to_string.rs b/crates/sail-function/src/scalar/spark_to_string.rs
index 882f7bc4e..c8b5f777b 100644
--- a/crates/sail-function/src/scalar/spark_to_string.rs
+++ b/crates/sail-function/src/scalar/spark_to_string.rs
@@ -11,7 +11,7 @@ use sail_common_datafusion::display::{ArrayFormatter, FormatOptions};
 use sail_common_datafusion::utils::items::ItemTaker;
 
 macro_rules! define_to_string_udf {
-    ($udf:ident, $name:expr, $return_type:expr, $func:expr $(,)?) => {
+    ($udf:ident, $name:expr_2021, $return_type:expr_2021, $func:expr_2021 $(,)?) => {
         #[derive(Debug, PartialEq, Eq, Hash)]
         pub struct $udf {
             signature: Signature,
diff --git a/crates/sail-function/src/scalar/string/decimal_format.rs b/crates/sail-function/src/scalar/string/decimal_format.rs
index c3320d2dc..dbd53a173 100644
--- a/crates/sail-function/src/scalar/string/decimal_format.rs
+++ b/crates/sail-function/src/scalar/string/decimal_format.rs
@@ -1,4 +1,4 @@
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 
 /// Spark's default pattern when an empty string is passed to format_number.
 const DEFAULT_PATTERN: &str = "#,###,###,###,###,###,##0";
diff --git a/crates/sail-function/src/scalar/string/format_number.rs b/crates/sail-function/src/scalar/string/format_number.rs
index 2b74c1416..4d43054a0 100644
--- a/crates/sail-function/src/scalar/string/format_number.rs
+++ b/crates/sail-function/src/scalar/string/format_number.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{ArrayRef, Float64Array, Int32Array, StringArray};
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::cast::as_float64_array;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use super::decimal_format::{format_with_parsed_pattern, insert_grouping, parse_pattern};
diff --git a/crates/sail-function/src/scalar/string/levenshtein.rs b/crates/sail-function/src/scalar/string/levenshtein.rs
index 70cfe52b9..0ef57344c 100644
--- a/crates/sail-function/src/scalar/string/levenshtein.rs
+++ b/crates/sail-function/src/scalar/string/levenshtein.rs
@@ -3,9 +3,9 @@ use std::sync::Arc;
 use datafusion::arrow::array::{Array, ArrayRef, Int32Array, Int64Array, OffsetSizeTrait};
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::cast::{as_generic_string_array, as_int32_array, as_string_view_array};
-use datafusion_common::types::{logical_int32, logical_string, NativeType};
+use datafusion_common::types::{NativeType, logical_int32, logical_string};
 use datafusion_common::utils::datafusion_strsim;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use datafusion_expr_common::signature::{Coercion, TypeSignature, TypeSignatureClass};
 use datafusion_expr_common::type_coercion::binary::{binary_to_string_coercion, string_coercion};
diff --git a/crates/sail-function/src/scalar/string/make_valid_utf8.rs b/crates/sail-function/src/scalar/string/make_valid_utf8.rs
index 821d927b9..3ffc63d6a 100644
--- a/crates/sail-function/src/scalar/string/make_valid_utf8.rs
+++ b/crates/sail-function/src/scalar/string/make_valid_utf8.rs
@@ -5,7 +5,7 @@ use datafusion::arrow::datatypes::DataType;
 use datafusion_common::cast::{
     as_binary_array, as_binary_view_array, as_fixed_size_binary_array, as_large_binary_array,
 };
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::function::Hint;
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use datafusion_functions::utils::make_scalar_function;
diff --git a/crates/sail-function/src/scalar/string/mod.rs b/crates/sail-function/src/scalar/string/mod.rs
index f073dc404..58668d982 100644
--- a/crates/sail-function/src/scalar/string/mod.rs
+++ b/crates/sail-function/src/scalar/string/mod.rs
@@ -7,6 +7,7 @@ pub mod soundex;
 pub mod spark_base64;
 pub mod spark_concat_ws;
 pub mod spark_encode_decode;
+pub mod spark_length;
 pub mod spark_mask;
 pub mod spark_quote;
 pub mod spark_regexp_extract_all;
diff --git a/crates/sail-function/src/scalar/string/randstr.rs b/crates/sail-function/src/scalar/string/randstr.rs
index b3e589fac..14cd4d98a 100644
--- a/crates/sail-function/src/scalar/string/randstr.rs
+++ b/crates/sail-function/src/scalar/string/randstr.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::array::StringArray;
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::scalar::math::xorshift::SparkXorShiftRandom;
@@ -59,7 +59,7 @@ impl ScalarUDFImpl for Randstr {
                 return exec_err!(
                     "`randstr` expects a constant integer length, got {:?}",
                     other.data_type()
-                )
+                );
             }
         };
 
@@ -75,7 +75,7 @@ impl ScalarUDFImpl for Randstr {
                     return exec_err!(
                         "`randstr` expects an integer seed, got {:?}",
                         other.data_type()
-                    )
+                    );
                 }
             }
         } else {
diff --git a/crates/sail-function/src/scalar/string/soundex.rs b/crates/sail-function/src/scalar/string/soundex.rs
index f8dc13efd..a40449735 100644
--- a/crates/sail-function/src/scalar/string/soundex.rs
+++ b/crates/sail-function/src/scalar/string/soundex.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{ArrayRef, GenericStringArray, OffsetSizeTrait, StringArray};
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::functions_utils::make_scalar_function;
diff --git a/crates/sail-function/src/scalar/string/spark_base64.rs b/crates/sail-function/src/scalar/string/spark_base64.rs
index fe9f33dd6..74cd21604 100644
--- a/crates/sail-function/src/scalar/string/spark_base64.rs
+++ b/crates/sail-function/src/scalar/string/spark_base64.rs
@@ -1,14 +1,15 @@
 use std::sync::Arc;
 
-use base64::engine::general_purpose::{GeneralPurpose, GeneralPurposeConfig, STANDARD};
 use base64::engine::DecodePaddingMode;
-use base64::{alphabet, Engine as _};
+use base64::engine::general_purpose::{GeneralPurpose, GeneralPurposeConfig, STANDARD};
+use base64::{Engine as _, alphabet};
 use datafusion::arrow::array::{
-    Array, BinaryArray, BinaryBuilder, BinaryViewArray, FixedSizeBinaryArray, LargeBinaryArray,
-    LargeBinaryBuilder, LargeStringArray, StringArray, StringViewArray,
+    Array, BinaryArray, BinaryViewArray, FixedSizeBinaryArray, GenericBinaryArray,
+    GenericBinaryBuilder, GenericStringArray, GenericStringBuilder, LargeBinaryArray,
+    LargeStringArray, OffsetSizeTrait, StringArray, StringViewArray,
 };
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_datafusion_err, exec_err, plan_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 const SPARK_BASE64_DECODE: GeneralPurpose = GeneralPurpose::new(
@@ -123,11 +124,13 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to BinaryArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i32,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i),
-                    ))
+                    ))))
                 }
                 DataType::BinaryView => {
                     let array = array
@@ -138,11 +141,13 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to BinaryViewArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i32,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i),
-                    ))
+                    ))))
                 }
                 DataType::FixedSizeBinary(_) => {
                     let array = array
@@ -153,11 +158,13 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to FixedSizeBinaryArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i32,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i),
-                    ))
+                    ))))
                 }
                 DataType::LargeBinary => {
                     let array = array
@@ -168,11 +175,13 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to LargeBinaryArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i64,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i),
-                    ))
+                    ))))
                 }
                 DataType::Utf8 => {
                     let array = array
@@ -183,11 +192,13 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to StringArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i32,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i).as_bytes(),
-                    ))
+                    ))))
                 }
                 DataType::LargeUtf8 => {
                     let array = array
@@ -198,11 +209,13 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to LargeStringArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i64,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i).as_bytes(),
-                    ))
+                    ))))
                 }
                 DataType::Utf8View => {
                     let array = array
@@ -213,33 +226,27 @@ impl ScalarUDFImpl for SparkBase64 {
                                 "Spark `base64`: Failed to downcast Expr to StringViewArray"
                             )
                         })?;
-                    Ok(encode_spark_base64_array(
+                    Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                        i32,
+                    >(
                         array.len(),
                         |i| array.is_null(i),
                         |i| array.value(i).as_bytes(),
-                    ))
+                    ))))
                 }
-                DataType::Null => Ok(encode_spark_base64_array(
+                DataType::Null => Ok(ColumnarValue::Array(Arc::new(encode_spark_base64_array::<
+                    i32,
+                >(
                     array.len(),
                     |_| true,
                     |_| &[],
-                )),
+                )))),
                 other => {
-                    exec_err!("Spark `base64`: Expr array must be BINARY or STRING, got array of type {other}")
-                }
-            }
-            .map(|results| match args[0].data_type() {
-                DataType::Utf8
-                | DataType::Utf8View
-                | DataType::Binary
-                | DataType::FixedSizeBinary(_)
-                | DataType::BinaryView => ColumnarValue::Array(Arc::new(StringArray::from(results))),
-                DataType::LargeUtf8 | DataType::LargeBinary => {
-                    ColumnarValue::Array(Arc::new(LargeStringArray::from(results)))
+                    exec_err!(
+                        "Spark `base64`: Expr array must be BINARY or STRING, got array of type {other}"
+                    )
                 }
-                DataType::Null => ColumnarValue::Array(Arc::new(StringArray::from(results))),
-                _ => unreachable!(),
-            }),
+            },
             other => exec_err!("Spark `base64`: Expr must be BINARY or STRING, got {other:?}"),
         }
     }
@@ -272,20 +279,23 @@ impl SparkUnbase64 {
     }
 }
 
-fn encode_spark_base64_array<'a>(
+fn encode_spark_base64_array<'a, O: OffsetSizeTrait>(
     len: usize,
     is_null: impl Fn(usize) -> bool,
     value: impl Fn(usize) -> &'a [u8],
-) -> Vec> {
-    let mut results = Vec::with_capacity(len);
+) -> GenericStringArray {
+    let mut builder = GenericStringBuilder::::with_capacity(len, 0);
+    let mut buf = String::new();
     for i in 0..len {
         if is_null(i) {
-            results.push(None);
+            builder.append_null();
         } else {
-            results.push(Some(STANDARD.encode(value(i))));
+            buf.clear();
+            STANDARD.encode_string(value(i), &mut buf);
+            builder.append_value(&buf);
         }
     }
-    results
+    builder.finish()
 }
 
 fn decode_spark_base64(value: &str) -> Result> {
@@ -313,20 +323,20 @@ fn is_spark_base64_byte(byte: u8) -> bool {
     byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=')
 }
 
-fn decode_spark_base64_array<'a>(
+fn decode_spark_base64_array<'a, O: OffsetSizeTrait>(
     len: usize,
     is_null: impl Fn(usize) -> bool,
     value: impl Fn(usize) -> &'a str,
-) -> Result>>> {
-    let mut results = Vec::with_capacity(len);
+) -> Result> {
+    let mut builder = GenericBinaryBuilder::::with_capacity(len, 0);
     for i in 0..len {
         if is_null(i) {
-            results.push(None);
+            builder.append_null();
         } else {
-            results.push(Some(decode_spark_base64(value(i))?));
+            builder.append_value(decode_spark_base64(value(i))?.as_slice());
         }
     }
-    Ok(results)
+    Ok(builder.finish())
 }
 
 impl ScalarUDFImpl for SparkUnbase64 {
@@ -394,7 +404,12 @@ impl ScalarUDFImpl for SparkUnbase64 {
                                 "Spark `unbase64`: Failed to downcast Expr to StringArray"
                             )
                         })?;
-                    decode_spark_base64_array(array.len(), |i| array.is_null(i), |i| array.value(i))
+                    decode_spark_base64_array::(
+                        array.len(),
+                        |i| array.is_null(i),
+                        |i| array.value(i),
+                    )
+                    .map(|a| ColumnarValue::Array(Arc::new(a)))
                 }
                 DataType::LargeUtf8 => {
                     let array = array
@@ -405,7 +420,12 @@ impl ScalarUDFImpl for SparkUnbase64 {
                                 "Spark `unbase64`: Failed to downcast Expr to LargeStringArray"
                             )
                         })?;
-                    decode_spark_base64_array(array.len(), |i| array.is_null(i), |i| array.value(i))
+                    decode_spark_base64_array::(
+                        array.len(),
+                        |i| array.is_null(i),
+                        |i| array.value(i),
+                    )
+                    .map(|a| ColumnarValue::Array(Arc::new(a)))
                 }
                 DataType::Utf8View => {
                     let array = array
@@ -416,37 +436,130 @@ impl ScalarUDFImpl for SparkUnbase64 {
                                 "Spark `unbase64`: Failed to downcast Expr to StringViewArray"
                             )
                         })?;
-                    decode_spark_base64_array(array.len(), |i| array.is_null(i), |i| array.value(i))
+                    decode_spark_base64_array::(
+                        array.len(),
+                        |i| array.is_null(i),
+                        |i| array.value(i),
+                    )
+                    .map(|a| ColumnarValue::Array(Arc::new(a)))
                 }
-                DataType::Null => decode_spark_base64_array(array.len(), |_| true, |_| ""),
+                DataType::Null => decode_spark_base64_array::(array.len(), |_| true, |_| "")
+                    .map(|a| ColumnarValue::Array(Arc::new(a))),
                 other => exec_err!(
                     "Spark `unbase64`: Expr array must be STRING, got array of type {other}"
                 ),
-            }
-            .and_then(|results| match args[0].data_type() {
-                DataType::Null | DataType::Utf8 | DataType::Utf8View => {
-                    let mut builder = BinaryBuilder::new();
-                    for value in results {
-                        match value {
-                            Some(value) => builder.append_value(value.as_slice()),
-                            None => builder.append_null(),
-                        }
-                    }
-                    Ok(ColumnarValue::Array(Arc::new(builder.finish())))
-                }
-                DataType::LargeUtf8 => {
-                    let mut builder = LargeBinaryBuilder::new();
-                    for value in results {
-                        match value {
-                            Some(value) => builder.append_value(value.as_slice()),
-                            None => builder.append_null(),
-                        }
-                    }
-                    Ok(ColumnarValue::Array(Arc::new(builder.finish())))
-                }
-                _ => plan_err!("1st argument should be String, got {}", args[0].data_type()),
-            }),
+            },
             other => exec_err!("Spark `unbase64`: Expr must be STRING, got {other:?}"),
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use datafusion::arrow::array::{BinaryArray, LargeBinaryArray, LargeStringArray, StringArray};
+    use datafusion::arrow::datatypes::Field;
+    use datafusion_common::config::ConfigOptions;
+
+    use super::*;
+
+    fn invoke(udf: &dyn ScalarUDFImpl, array: Arc) -> Result> {
+        let field = Arc::new(Field::new("v", array.data_type().clone(), true));
+        let number_rows = array.len();
+        let return_type = udf.return_type(&[array.data_type().clone()])?;
+        let result = udf.invoke_with_args(ScalarFunctionArgs {
+            args: vec![ColumnarValue::Array(array)],
+            arg_fields: vec![Arc::clone(&field)],
+            number_rows,
+            return_field: Arc::new(Field::new("r", return_type, true)),
+            config_options: Arc::new(ConfigOptions::default()),
+        })?;
+        result.into_array(number_rows)
+    }
+
+    #[test]
+    fn test_base64_binary_array_with_nulls_and_empty() -> Result<()> {
+        let input: Arc = Arc::new(BinaryArray::from_opt_vec(vec![
+            Some(b"hi".as_ref()),
+            None,
+            Some(b"".as_ref()),
+        ]));
+        let out = invoke(&SparkBase64::new(), input)?;
+        assert_eq!(out.data_type(), &DataType::Utf8);
+        let out = out
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| exec_datafusion_err!("expected StringArray"))?;
+        assert_eq!(out.value(0), "aGk=");
+        assert!(out.is_null(1));
+        assert_eq!(out.value(2), "");
+        Ok(())
+    }
+
+    #[test]
+    fn test_base64_large_binary_array_returns_large_utf8() -> Result<()> {
+        let input: Arc = Arc::new(LargeBinaryArray::from_opt_vec(vec![
+            Some(b"hi".as_ref()),
+            None,
+        ]));
+        let out = invoke(&SparkBase64::new(), input)?;
+        assert_eq!(out.data_type(), &DataType::LargeUtf8);
+        let out = out
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| exec_datafusion_err!("expected LargeStringArray"))?;
+        assert_eq!(out.value(0), "aGk=");
+        assert!(out.is_null(1));
+        Ok(())
+    }
+
+    #[test]
+    fn test_unbase64_utf8_array_with_nulls_empty_and_invalid() -> Result<()> {
+        let input: Arc = Arc::new(StringArray::from(vec![
+            Some("aGk="),
+            None,
+            Some(""),
+            Some("%"),
+        ]));
+        let out = invoke(&SparkUnbase64::new(), input)?;
+        assert_eq!(out.data_type(), &DataType::Binary);
+        let out = out
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| exec_datafusion_err!("expected BinaryArray"))?;
+        assert_eq!(out.value(0), b"hi");
+        assert!(out.is_null(1));
+        assert_eq!(out.value(2), b"");
+        assert_eq!(out.value(3), b"");
+        Ok(())
+    }
+
+    #[test]
+    fn test_unbase64_large_utf8_array_returns_large_binary() -> Result<()> {
+        let input: Arc = Arc::new(LargeStringArray::from(vec![Some("aGk="), None]));
+        let out = invoke(&SparkUnbase64::new(), input)?;
+        assert_eq!(out.data_type(), &DataType::LargeBinary);
+        let out = out
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| exec_datafusion_err!("expected LargeBinaryArray"))?;
+        assert_eq!(out.value(0), b"hi");
+        assert!(out.is_null(1));
+        Ok(())
+    }
+
+    #[test]
+    fn test_base64_unbase64_round_trip() -> Result<()> {
+        let input: Arc =
+            Arc::new(StringArray::from(vec![Some("foo"), Some(""), Some("bar")]));
+        let encoded = invoke(&SparkBase64::new(), input)?;
+        let decoded = invoke(&SparkUnbase64::new(), encoded)?;
+        let decoded = decoded
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| exec_datafusion_err!("expected BinaryArray"))?;
+        assert_eq!(decoded.value(0), b"foo");
+        assert_eq!(decoded.value(1), b"");
+        assert_eq!(decoded.value(2), b"bar");
+        Ok(())
+    }
+}
diff --git a/crates/sail-function/src/scalar/string/spark_concat_ws.rs b/crates/sail-function/src/scalar/string/spark_concat_ws.rs
index 71cdf2722..668e50398 100644
--- a/crates/sail-function/src/scalar/string/spark_concat_ws.rs
+++ b/crates/sail-function/src/scalar/string/spark_concat_ws.rs
@@ -11,12 +11,15 @@
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    new_null_array, Array, ArrayRef, AsArray, GenericListArray, LargeStringArray, OffsetSizeTrait,
-    StringArray, StringBuilder, StringViewArray,
+    Array, ArrayRef, AsArray, GenericListArray, LargeStringArray, OffsetSizeTrait, StringArray,
+    StringBuilder, StringViewArray, new_null_array,
+};
+use datafusion::arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
+use datafusion_expr::{
+    ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
-use datafusion::arrow::datatypes::DataType;
-use datafusion_common::Result;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err};
 use crate::functions_utils::make_scalar_function;
@@ -50,7 +53,68 @@ impl ScalarUDFImpl for SparkConcatWs {
     }
 
     fn return_type(&self, _arg_types: &[DataType]) -> Result {
-        Ok(DataType::Utf8)
+        internal_err!(
+            "spark_concat_ws: `return_type` should not be called; `return_field_from_args` is used instead"
+        )
+    }
+
+    /// `concat_ws` is null only when the separator is null; null value arguments
+    /// are skipped, so a non-null separator always yields a (possibly empty)
+    /// string. Mirrors Spark's `ConcatWs.nullable = children.head.nullable`.
+    ///
+    /// Set here (not in the deprecated `is_nullable`, which DataFusion no longer
+    /// consults to derive the output field).
+    fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result {
+        let nullable = args.arg_fields.first().is_none_or(|sep| sep.is_nullable());
+        Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable)))
+    }
+
+    /// Plan-time simplifications that mirror Spark's `concat_ws` semantics, so the
+    /// per-row kernel runs on a smaller (or no) argument set.
+    ///
+    /// concat_ws — simplify summary:
+    /// ```text
+    /// concat_ws(NULL, …)             -> NULL                  (R1: null literal separator)
+    /// concat_ws(sep, a, NULL, b)     -> concat_ws(sep, a, b)  (R2: drop null value literals)
+    /// concat_ws()  -> ''                    (R3: separator only)
+    /// ```
+    ///
+    /// * **R1** — a null separator literal makes every row null regardless of the
+    ///   value arguments (even non-literal columns), so the whole call folds to a
+    ///   typed null. Runs first: `concat_ws(NULL)` is NULL, not `''`.
+    /// * **R2** — Spark silently skips null value arguments, so a null literal among
+    ///   the values (index >= 1) is dropped. This also reaches the mixed case
+    ///   `concat_ws(col, a, NULL, b)` that constant folding cannot, since the call
+    ///   is not all-literal.
+    /// * **R3** — `concat_ws(sep)` with a non-null literal separator and no value
+    ///   arguments is always the empty string. A *column* separator is left
+    ///   untouched: the kernel still has to emit null for its null rows.
+    fn simplify(&self, args: Vec, _info: &SimplifyContext) -> Result {
+        if matches!(args.first(), Some(Expr::Literal(scalar, _)) if scalar.is_null()) {
+            return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
+                ScalarValue::Utf8(None),
+                None,
+            )));
+        }
+
+        // Keep the separator (index 0); drop null literals among the values.
+        let args: Vec = args
+            .into_iter()
+            .enumerate()
+            .filter(|(i, arg)| {
+                *i == 0 || !matches!(arg, Expr::Literal(scalar, _) if scalar.is_null())
+            })
+            .map(|(_, arg)| arg)
+            .collect();
+
+        if matches!(args.as_slice(), [Expr::Literal(_, _)]) {
+            return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
+                ScalarValue::Utf8(Some(String::new())),
+                None,
+            )));
+        }
+
+        Ok(ExprSimplifyResult::Original(args))
     }
 
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
@@ -116,7 +180,7 @@ fn concat_ws_inner(args: &[ArrayRef]) -> Result {
         .map(ConcatArg::try_new)
         .collect::>>()?;
 
-    let mut builder = StringBuilder::with_capacity(num_rows, num_rows * 16);
+    let mut builder = StringBuilder::with_capacity(num_rows, 0);
     let mut row_buf = String::new(); // reused across rows; capacity is kept after clear()
 
     for row in 0..num_rows {
diff --git a/crates/sail-function/src/scalar/string/spark_encode_decode.rs b/crates/sail-function/src/scalar/string/spark_encode_decode.rs
index 4c653a7a4..427a6afe0 100644
--- a/crates/sail-function/src/scalar/string/spark_encode_decode.rs
+++ b/crates/sail-function/src/scalar/string/spark_encode_decode.rs
@@ -5,7 +5,7 @@ use datafusion::arrow::array::{
     GenericStringBuilder, LargeBinaryBuilder, OffsetSizeTrait, StringArrayType,
 };
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_datafusion_err, exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err};
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, Volatility};
@@ -114,7 +114,7 @@ impl ScalarUDFImpl for SparkEncode {
                                 let string_array = string_array.as_string::();
                                 let mut builder = BinaryBuilder::with_capacity(
                                     string_array.len(),
-                                    string_array.len(),
+                                    string_array.get_buffer_memory_size(),
                                 );
                                 for string in string_array.iter() {
                                     if let Some(string) = string {
@@ -129,7 +129,7 @@ impl ScalarUDFImpl for SparkEncode {
                                 let string_array = string_array.as_string::();
                                 let mut builder = LargeBinaryBuilder::with_capacity(
                                     string_array.len(),
-                                    string_array.len(),
+                                    string_array.get_buffer_memory_size(),
                                 );
                                 for string in string_array.iter() {
                                     if let Some(string) = string {
@@ -144,7 +144,7 @@ impl ScalarUDFImpl for SparkEncode {
                                 let string_array = string_array.as_string_view();
                                 let mut builder = BinaryBuilder::with_capacity(
                                     string_array.len(),
-                                    string_array.len(),
+                                    string_array.get_buffer_memory_size(),
                                 );
                                 for string in string_array.iter() {
                                     if let Some(string) = string {
@@ -156,7 +156,9 @@ impl ScalarUDFImpl for SparkEncode {
                                 Ok(Arc::new(builder.finish()) as ArrayRef)
                             }
                             other => {
-                                exec_err!("Spark `encode` function: First arg must be STRING, got {other:?}")
+                                exec_err!(
+                                    "Spark `encode` function: First arg must be STRING, got {other:?}"
+                                )
                             }
                         }?;
                         Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef))
@@ -333,16 +335,16 @@ impl ScalarUDFImpl for SparkDecode {
                         }
                     }
                     _ => {
-                        exec_err!("Spark `decode` function: First arg must be BINARY and second arg must be STRING type, got {args:?}")
+                        exec_err!(
+                            "Spark `decode` function: First arg must be BINARY and second arg must be STRING type, got {args:?}"
+                        )
                     }
                 }
             }
             (ColumnarValue::Array(binary_array), ColumnarValue::Scalar(char_set)) => {
                 match (binary_array.data_type(), char_set) {
                     (
-                        DataType::Binary
-                        | DataType::LargeBinary
-                        | DataType::BinaryView,
+                        DataType::Binary | DataType::LargeBinary | DataType::BinaryView,
                         ScalarValue::Utf8(Some(char_set))
                         | ScalarValue::LargeUtf8(Some(char_set))
                         | ScalarValue::Utf8View(Some(char_set)),
@@ -388,12 +390,16 @@ impl ScalarUDFImpl for SparkDecode {
                                 Ok(Arc::new(builder.finish()) as ArrayRef)
                             }
                             other => {
-                                exec_err!("Spark `decode` function: First arg must be BINARY, got {other:?}")
+                                exec_err!(
+                                    "Spark `decode` function: First arg must be BINARY, got {other:?}"
+                                )
                             }
                         }?;
                         Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef))
                     }
-                    _ => exec_err!("Spark `decode` function: First arg must be BINARY and second arg must be STRING type, got {args:?}"),
+                    _ => exec_err!(
+                        "Spark `decode` function: First arg must be BINARY and second arg must be STRING type, got {args:?}"
+                    ),
                 }
             }
             (ColumnarValue::Array(binary_array), ColumnarValue::Array(char_set_array)) => {
@@ -488,7 +494,8 @@ fn encode(string: &str, char_set: &str) -> Result> {
         "US-ASCII" => {
             // 'US-ASCII': Seven-bit ASCII, ISO646-US.
             let bytes = if !string.is_ascii() {
-                string.chars()
+                string
+                    .chars()
                     .map(|c| if c.is_ascii() { c as u8 } else { b'?' })
                     .collect()
             } else {
@@ -498,7 +505,8 @@ fn encode(string: &str, char_set: &str) -> Result> {
         }
         "ISO-8859-1" => {
             // 'ISO-8859-1': ISO Latin Alphabet No. 1, ISO-LATIN-1.
-            Ok(string.chars()
+            Ok(string
+                .chars()
                 .map(|c| if c as u32 <= 0xFF { c as u8 } else { b'?' })
                 .collect())
         }
@@ -531,7 +539,9 @@ fn encode(string: &str, char_set: &str) -> Result> {
             }
             Ok(bytes)
         }
-        _ => exec_err!("Spark `encode` function: Unsupported charset {char_set}. Charset must be one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'"),
+        _ => exec_err!(
+            "Spark `encode` function: Unsupported charset {char_set}. Charset must be one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'"
+        ),
     }
 }
 
@@ -563,8 +573,11 @@ fn decode(bytes: &[u8], char_set: &str) -> Result {
             if !bytes.is_ascii() {
                 exec_err!("Spark `decode` function: Invalid US-ASCII byte sequence: {bytes:?}")
             } else {
-                String::from_utf8(bytes.to_vec())
-                    .map_err(|e| exec_datafusion_err!("Spark `decode` function: Failed to decode US-ASCII bytes: {e}"))
+                String::from_utf8(bytes.to_vec()).map_err(|e| {
+                    exec_datafusion_err!(
+                        "Spark `decode` function: Failed to decode US-ASCII bytes: {e}"
+                    )
+                })
             }
         }
         "ISO-8859-1" => {
@@ -573,20 +586,26 @@ fn decode(bytes: &[u8], char_set: &str) -> Result {
         }
         "UTF-8" => {
             // 'UTF-8': Eight-bit UCS Transformation Format.
-            String::from_utf8(bytes.to_vec())
-                .map_err(|e| exec_datafusion_err!("Spark `decode` function: Failed to decode UTF-8 bytes: {e}"))
+            String::from_utf8(bytes.to_vec()).map_err(|e| {
+                exec_datafusion_err!("Spark `decode` function: Failed to decode UTF-8 bytes: {e}")
+            })
         }
         "UTF-16BE" => {
             // 'UTF-16BE': Sixteen-bit UCS Transformation Format, big-endian byte order.
             if !bytes.len().is_multiple_of(2) {
                 exec_err!("Spark `decode` function: Invalid UTF-16BE byte sequence: {bytes:?}")
             } else {
-                let u16_words: Vec = bytes
-                    .chunks_exact(2)
-                    .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
+                let (chunks, remainder) = bytes.as_chunks::<2>();
+                debug_assert!(remainder.is_empty());
+                let u16_words: Vec = chunks
+                    .iter()
+                    .map(|chunk| u16::from_be_bytes(*chunk))
                     .collect();
-                String::from_utf16(&u16_words)
-                    .map_err(|e| exec_datafusion_err!("Spark `decode` function: Failed to decode UTF-16BE bytes: {e}"))
+                String::from_utf16(&u16_words).map_err(|e| {
+                    exec_datafusion_err!(
+                        "Spark `decode` function: Failed to decode UTF-16BE bytes: {e}"
+                    )
+                })
             }
         }
         "UTF-16LE" => {
@@ -594,12 +613,17 @@ fn decode(bytes: &[u8], char_set: &str) -> Result {
             if !bytes.len().is_multiple_of(2) {
                 exec_err!("Spark `decode` function: Invalid UTF-16LE byte sequence: {bytes:?}")
             } else {
-                let u16_words: Vec = bytes
-                    .chunks_exact(2)
-                    .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
+                let (chunks, remainder) = bytes.as_chunks::<2>();
+                debug_assert!(remainder.is_empty());
+                let u16_words: Vec = chunks
+                    .iter()
+                    .map(|chunk| u16::from_le_bytes(*chunk))
                     .collect();
-                String::from_utf16(&u16_words)
-                    .map_err(|e| exec_datafusion_err!("Spark `decode` function: Failed to decode UTF-16LE bytes: {e}"))
+                String::from_utf16(&u16_words).map_err(|e| {
+                    exec_datafusion_err!(
+                        "Spark `decode` function: Failed to decode UTF-16LE bytes: {e}"
+                    )
+                })
             }
         }
         "UTF-16" => {
@@ -608,28 +632,37 @@ fn decode(bytes: &[u8], char_set: &str) -> Result {
                 exec_err!("Spark `decode` function: Invalid UTF-16 byte sequence: {bytes:?}")
             } else {
                 let (bom, content) = match &bytes[0..2] {
-                    [0xFE, 0xFF] => (true, &bytes[2..]), // BE BOM
+                    [0xFE, 0xFF] => (true, &bytes[2..]),  // BE BOM
                     [0xFF, 0xFE] => (false, &bytes[2..]), // LE BOM
-                    _ => (true, bytes), // Default to BE if no BOM
+                    _ => (true, bytes),                   // Default to BE if no BOM
                 };
 
                 if !content.len().is_multiple_of(2) {
                     exec_err!("Spark `decode` function: Invalid UTF-16 byte sequence: {bytes:?}")
                 } else {
-                    let u16_words: Vec = content
-                        .chunks_exact(2)
-                        .map(|chunk| if bom {
-                            u16::from_be_bytes([chunk[0], chunk[1]])
-                        } else {
-                            u16::from_le_bytes([chunk[0], chunk[1]])
+                    let (chunks, remainder) = content.as_chunks::<2>();
+                    debug_assert!(remainder.is_empty());
+                    let u16_words: Vec = chunks
+                        .iter()
+                        .map(|chunk| {
+                            if bom {
+                                u16::from_be_bytes(*chunk)
+                            } else {
+                                u16::from_le_bytes(*chunk)
+                            }
                         })
                         .collect();
-                    String::from_utf16(&u16_words)
-                        .map_err(|e| exec_datafusion_err!("Spark `decode` function: Failed to decode UTF-16 bytes: {e}"))
+                    String::from_utf16(&u16_words).map_err(|e| {
+                        exec_datafusion_err!(
+                            "Spark `decode` function: Failed to decode UTF-16 bytes: {e}"
+                        )
+                    })
                 }
             }
         }
-        _ => exec_err!("Spark `decode` function: Unsupported charset {char_set}. Charset must be one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'"),
+        _ => exec_err!(
+            "Spark `decode` function: Unsupported charset {char_set}. Charset must be one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'"
+        ),
     }
 }
 
diff --git a/crates/sail-function/src/scalar/string/spark_length.rs b/crates/sail-function/src/scalar/string/spark_length.rs
new file mode 100644
index 000000000..0a9efe860
--- /dev/null
+++ b/crates/sail-function/src/scalar/string/spark_length.rs
@@ -0,0 +1,99 @@
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, PrimitiveArray};
+use datafusion::arrow::compute::kernels::length::{bit_length, length};
+use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Int32Type, Int64Type};
+use datafusion_common::Result;
+use datafusion_expr::{
+    ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_functions::utils::make_scalar_function;
+
+use crate::error::generic_internal_err;
+
+/// Spark measures the byte length of both string and binary data, unlike DataFusion's
+/// `octet_length` and `bit_length`, whose signatures accept string data only. Feeding them
+/// binary data coerces it to UTF-8 first, which rejects data that is not valid UTF-8.
+///
+/// The Arrow kernels already measure both, from the offset buffer alone, and already wrap
+/// on overflow the way Spark does. They return `Int64` for the large variants, whereas Spark
+/// always returns `Int32`.
+macro_rules! define_length_udf {
+    ($udf:ident, $name:expr_2021, $kernel:expr_2021) => {
+        #[derive(Debug, PartialEq, Eq, Hash)]
+        pub struct $udf {
+            signature: Signature,
+        }
+
+        impl Default for $udf {
+            fn default() -> Self {
+                Self::new()
+            }
+        }
+
+        impl $udf {
+            pub fn new() -> Self {
+                Self {
+                    signature: Signature::uniform(
+                        1,
+                        vec![
+                            DataType::Utf8,
+                            DataType::LargeUtf8,
+                            DataType::Utf8View,
+                            DataType::Binary,
+                            DataType::LargeBinary,
+                            DataType::BinaryView,
+                        ],
+                        Volatility::Immutable,
+                    ),
+                }
+            }
+        }
+
+        impl ScalarUDFImpl for $udf {
+            fn name(&self) -> &str {
+                $name
+            }
+
+            fn signature(&self) -> &Signature {
+                &self.signature
+            }
+
+            fn return_type(&self, _arg_types: &[DataType]) -> Result {
+                Err(generic_internal_err(
+                    $name,
+                    "`return_type` should not be called, call `return_field_from_args` instead",
+                ))
+            }
+
+            fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result {
+                let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+                Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable)))
+            }
+
+            fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+                make_scalar_function(|args: &[ArrayRef]| into_int32($kernel(&args[0])?), vec![])(
+                    &args.args,
+                )
+            }
+        }
+    };
+}
+
+define_length_udf!(SparkOctetLength, "spark_octet_length", length);
+define_length_udf!(SparkBitLength, "spark_bit_length", bit_length);
+
+/// The large variants measure into `Int64`; Spark returns `Int32` for every input type.
+/// The conversion truncates rather than nullifying, so a value whose length overflows
+/// `Int32` wraps, which is what Spark does.
+fn into_int32(lengths: ArrayRef) -> Result {
+    match lengths.data_type() {
+        DataType::Int64 => {
+            let lengths: PrimitiveArray = lengths
+                .as_primitive::()
+                .unary(|length| length as i32);
+            Ok(Arc::new(lengths))
+        }
+        _ => Ok(lengths),
+    }
+}
diff --git a/crates/sail-function/src/scalar/string/spark_mask.rs b/crates/sail-function/src/scalar/string/spark_mask.rs
index 030038b92..ac2151c0a 100644
--- a/crates/sail-function/src/scalar/string/spark_mask.rs
+++ b/crates/sail-function/src/scalar/string/spark_mask.rs
@@ -1,8 +1,8 @@
 use std::sync::Arc;
 
-use datafusion::arrow::array::{new_null_array, ArrayRef, AsArray, GenericStringBuilder};
+use datafusion::arrow::array::{ArrayRef, AsArray, GenericStringBuilder, new_null_array};
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, Volatility};
@@ -87,7 +87,9 @@ impl ScalarUDFImpl for SparkMask {
                         if string.chars().count() == 1 {
                             Ok(string.chars().next())
                         } else {
-                            exec_err!("Spark `mask` function: second arg must be a single character, got {string}")
+                            exec_err!(
+                                "Spark `mask` function: second arg must be a single character, got {string}"
+                            )
                         }
                     } else {
                         Ok(None)
@@ -112,7 +114,9 @@ impl ScalarUDFImpl for SparkMask {
                         if string.chars().count() == 1 {
                             Ok(string.chars().next())
                         } else {
-                            exec_err!("Spark `mask` function: third arg must be a single character, got {string}")
+                            exec_err!(
+                                "Spark `mask` function: third arg must be a single character, got {string}"
+                            )
                         }
                     } else {
                         Ok(None)
@@ -137,7 +141,9 @@ impl ScalarUDFImpl for SparkMask {
                         if string.chars().count() == 1 {
                             Ok(string.chars().next())
                         } else {
-                            exec_err!("Spark `mask` function: fourth arg must be a single character, got {string}")
+                            exec_err!(
+                                "Spark `mask` function: fourth arg must be a single character, got {string}"
+                            )
                         }
                     } else {
                         Ok(None)
@@ -162,7 +168,9 @@ impl ScalarUDFImpl for SparkMask {
                         if string.chars().count() == 1 {
                             Ok(string.chars().next())
                         } else {
-                            exec_err!("Spark `mask` function: fifth arg must be a single character, got {string}")
+                            exec_err!(
+                                "Spark `mask` function: fifth arg must be a single character, got {string}"
+                            )
                         }
                     } else {
                         Ok(None)
diff --git a/crates/sail-function/src/scalar/string/spark_quote.rs b/crates/sail-function/src/scalar/string/spark_quote.rs
index 30afe67ec..766dc9544 100644
--- a/crates/sail-function/src/scalar/string/spark_quote.rs
+++ b/crates/sail-function/src/scalar/string/spark_quote.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{ArrayRef, GenericStringArray, OffsetSizeTrait, StringArray};
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::functions_utils::make_scalar_function;
diff --git a/crates/sail-function/src/scalar/string/spark_split.rs b/crates/sail-function/src/scalar/string/spark_split.rs
index 2ecaa99c3..6f282c612 100644
--- a/crates/sail-function/src/scalar/string/spark_split.rs
+++ b/crates/sail-function/src/scalar/string/spark_split.rs
@@ -55,10 +55,10 @@ impl ScalarUDFImpl for SparkSplit {
     fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
         let err = || {
             Err(unsupported_data_types_exec_err(
-            Self::NAME,
-            "Expected (STRING, STRING) or (STRING, STRING, INT). Adjust the value to match the syntax, or change its target type. Use try_cast to handle malformed input and return NULL instead",
-            arg_types,
-        ))
+                Self::NAME,
+                "Expected (STRING, STRING) or (STRING, STRING, INT). Adjust the value to match the syntax, or change its target type. Use try_cast to handle malformed input and return NULL instead",
+                arg_types,
+            ))
         };
 
         let mut res_types = vec![];
diff --git a/crates/sail-function/src/scalar/string/spark_to_binary.rs b/crates/sail-function/src/scalar/string/spark_to_binary.rs
index 1d04476db..b70b6235c 100644
--- a/crates/sail-function/src/scalar/string/spark_to_binary.rs
+++ b/crates/sail-function/src/scalar/string/spark_to_binary.rs
@@ -3,9 +3,9 @@ use std::sync::Arc;
 use datafusion::arrow::datatypes::DataType;
 use datafusion::functions::encoding::expr_fn::decode;
 use datafusion::functions::encoding::inner::DecodeFunc;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
-use datafusion_expr::{expr, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl};
+use datafusion_expr::{Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, expr};
 use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, TypeSignature, Volatility};
 use sail_common_datafusion::utils::items::ItemTaker;
diff --git a/crates/sail-function/src/scalar/string/spark_to_char.rs b/crates/sail-function/src/scalar/string/spark_to_char.rs
index b5d757748..2ae1cbbba 100644
--- a/crates/sail-function/src/scalar/string/spark_to_char.rs
+++ b/crates/sail-function/src/scalar/string/spark_to_char.rs
@@ -4,9 +4,9 @@ use datafusion::arrow::array::{
     Array, ArrayRef, Decimal128Array, Decimal256Array, Float32Array, Float64Array, Int64Array,
     StringArray, UInt64Array,
 };
-use datafusion::arrow::compute::{cast_with_options, CastOptions};
-use datafusion::arrow::datatypes::{i256, DataType};
-use datafusion_common::{exec_err, internal_err, Result};
+use datafusion::arrow::compute::{CastOptions, cast_with_options};
+use datafusion::arrow::datatypes::{DataType, i256};
+use datafusion_common::{Result, exec_err, internal_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err};
@@ -412,7 +412,7 @@ struct NumberFormat {
 }
 
 macro_rules! invalid_format_err {
-    ($sub_class:expr, $format:expr, $($message:tt)*) => {
+    ($sub_class:expr_2021, $format:expr_2021, $($message:tt)*) => {
         exec_err!(
             "[INVALID_FORMAT.{}] The format is invalid: '{}'. {}",
             $sub_class,
@@ -531,10 +531,10 @@ impl NumberFormat {
                 FormatToken::DecimalPoint => {
                     // If the last character so far is a space, change it to a zero:
                     // the value has no integer part.
-                    if let Some(last) = result.last_mut() {
-                        if *last == b' ' {
-                            *last = b'0';
-                        }
+                    if let Some(last) = result.last_mut()
+                        && *last == b' '
+                    {
+                        *last = b'0';
                     }
                     result.push(b'.');
                     reached_decimal_point = true;
@@ -859,9 +859,9 @@ fn add_space_checking_trailing_characters(result: &mut Vec) {
 /// If the result contains a decimal point with nothing after it, replace the decimal
 /// point with a space.
 fn strip_trailing_lone_decimal_point(result: &mut [u8]) {
-    if let Some(i) = result.iter().position(|c| *c == b'.') {
-        if i == result.len() - 1 || result[i + 1] == b' ' {
-            result[i] = b' ';
-        }
+    if let Some(i) = result.iter().position(|c| *c == b'.')
+        && (i == result.len() - 1 || result[i + 1] == b' ')
+    {
+        result[i] = b' ';
     }
 }
diff --git a/crates/sail-function/src/scalar/string/spark_to_number.rs b/crates/sail-function/src/scalar/string/spark_to_number.rs
index 3b4f2c22f..3ca96391c 100644
--- a/crates/sail-function/src/scalar/string/spark_to_number.rs
+++ b/crates/sail-function/src/scalar/string/spark_to_number.rs
@@ -6,7 +6,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{ArrayRef, StringArray, *};
 use datafusion::arrow::datatypes::{DataType, *};
 use datafusion_common::{
-    exec_datafusion_err, exec_err, internal_err, DataFusionError, Result, ScalarValue,
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,
 };
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -257,7 +257,7 @@ impl ParsedNumber {
 }
 
 macro_rules! get_opt_capture_group {
-    ($captures:expr, $group_name:expr) => {
+    ($captures:expr_2021, $group_name:expr_2021) => {
         $captures
             .name($group_name)
             .map(|m| m.as_str().trim().to_string())
@@ -265,7 +265,7 @@ macro_rules! get_opt_capture_group {
 }
 
 macro_rules! get_capture_group {
-    ($captures:expr, $group_name:expr) => {
+    ($captures:expr_2021, $group_name:expr_2021) => {
         get_opt_capture_group!($captures, $group_name).ok_or_else(|| {
             exec_datafusion_err!(
                 "Missing '{}' group in the format regex or not well formed.",
@@ -280,13 +280,13 @@ macro_rules! get_capture_group {
 // is only inspected (compared, measured), never stored — avoiding an allocation
 // per row per group.
 macro_rules! get_opt_capture_str {
-    ($captures:expr, $group_name:expr) => {
+    ($captures:expr_2021, $group_name:expr_2021) => {
         $captures.name($group_name).map(|m| m.as_str().trim())
     };
 }
 
 macro_rules! get_capture_str {
-    ($captures:expr, $group_name:expr) => {
+    ($captures:expr_2021, $group_name:expr_2021) => {
         get_opt_capture_str!($captures, $group_name).ok_or_else(|| {
             exec_datafusion_err!(
                 "Missing '{}' group in the format regex or not well formed.",
@@ -375,8 +375,8 @@ fn validate_thousands_separator_positions(numbers: &str) -> Result<()> {
             "The format is invalid: '{numbers}'. Thousands separators (, or G) must have digits on both sides."
         );
     }
-    for w in bytes.windows(2) {
-        if is_sep(w[0]) && is_sep(w[1]) {
+    for &[left, right] in bytes.array_windows::<2>() {
+        if is_sep(left) && is_sep(right) {
             return exec_err!(
                 "The format is invalid: '{numbers}'. Thousands separators (, or G) must have digits between them."
             );
@@ -644,7 +644,9 @@ fn match_grouping(value_captures: &Captures, format_spec: &RegexSpec) -> Result<
     let has_comma = format_positions.iter().any(|(_, d)| *d == ',');
     let has_g = format_positions.iter().any(|(_, d)| *d == 'G');
     if has_comma && has_g {
-        return exec_err!("Malformed integer format related groupings: {format_numbers}. Cannot mix ',' and 'G' separators.");
+        return exec_err!(
+            "Malformed integer format related groupings: {format_numbers}. Cannot mix ',' and 'G' separators."
+        );
     }
 
     // Check if the groupings are in the correct order position
diff --git a/crates/sail-function/src/scalar/struct_function.rs b/crates/sail-function/src/scalar/struct_function.rs
index f13fda1c4..f129629c4 100644
--- a/crates/sail-function/src/scalar/struct_function.rs
+++ b/crates/sail-function/src/scalar/struct_function.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use datafusion::arrow::array::{ArrayRef, StructArray};
 use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Fields};
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
@@ -104,13 +104,17 @@ impl ScalarUDFImpl for StructFunction {
 
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
         let ScalarFunctionArgs {
-            args, arg_fields, ..
+            args, return_field, ..
         } = args;
+        let arg_fields = match return_field.data_type() {
+            DataType::Struct(fields) => fields.iter().cloned().collect::>(),
+            other => return exec_err!("struct: expected struct return field, got {}", other),
+        };
         let arrays = ColumnarValue::values_to_arrays(&args)?;
         Ok(ColumnarValue::Array(to_struct_array(
             arrays.as_slice(),
             self.field_names.as_slice(),
-            &arg_fields,
+            arg_fields.as_slice(),
         )?))
     }
 }
diff --git a/crates/sail-function/src/scalar/table_input.rs b/crates/sail-function/src/scalar/table_input.rs
index 5eff087aa..3d4dbb7bc 100644
--- a/crates/sail-function/src/scalar/table_input.rs
+++ b/crates/sail-function/src/scalar/table_input.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use datafusion::arrow::datatypes::DataType;
-use datafusion_common::{plan_err, Result};
+use datafusion_common::{Result, plan_err};
 use datafusion_expr::{
     ColumnarValue, LogicalPlan, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
diff --git a/crates/sail-function/src/scalar/update_struct_field.rs b/crates/sail-function/src/scalar/update_struct_field.rs
index 9252b033e..a898cbdaa 100644
--- a/crates/sail-function/src/scalar/update_struct_field.rs
+++ b/crates/sail-function/src/scalar/update_struct_field.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{Array, ArrayRef, StructArray};
 use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion_common::cast::as_struct_array;
-use datafusion_common::{exec_datafusion_err, exec_err, plan_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 
 #[derive(Debug, PartialEq, Eq, Hash)]
diff --git a/crates/sail-function/src/scalar/url/parse_url.rs b/crates/sail-function/src/scalar/url/parse_url.rs
index 281038b65..16f0cfa60 100644
--- a/crates/sail-function/src/scalar/url/parse_url.rs
+++ b/crates/sail-function/src/scalar/url/parse_url.rs
@@ -4,7 +4,7 @@ use datafusion::arrow::array::{
     Array, ArrayRef, GenericStringBuilder, LargeStringArray, StringArray, StringArrayType,
 };
 use datafusion::arrow::datatypes::DataType;
-use datafusion::common::{exec_datafusion_err, exec_err, plan_err, Result};
+use datafusion::common::{Result, exec_datafusion_err, exec_err, plan_err};
 use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use url::Url;
@@ -252,7 +252,7 @@ pub fn spark_handled_parse_url(
     let url = &args[0];
     let part = &args[1];
 
-    let result = if args.len() == 3 {
+    if args.len() == 3 {
         // In this case, the 'key' argument is passed
         let key = &args[2];
 
@@ -557,8 +557,7 @@ pub fn spark_handled_parse_url(
             }
             _ => exec_err!("{} expects STRING arguments, got {:?}", "`parse_url`", args),
         }
-    };
-    result
+    }
 }
 
 fn process_parse_url<'a, A, B, C, T>(
diff --git a/crates/sail-function/src/scalar/url/spark_try_parse_url.rs b/crates/sail-function/src/scalar/url/spark_try_parse_url.rs
index 3889baf78..21a8f407f 100644
--- a/crates/sail-function/src/scalar/url/spark_try_parse_url.rs
+++ b/crates/sail-function/src/scalar/url/spark_try_parse_url.rs
@@ -6,7 +6,7 @@ use datafusion_expr_common::columnar_value::ColumnarValue;
 use datafusion_expr_common::signature::{Signature, Volatility};
 
 use crate::functions_utils::make_scalar_function;
-use crate::scalar::url::parse_url::{spark_handled_parse_url, ParseUrl};
+use crate::scalar::url::parse_url::{ParseUrl, spark_handled_parse_url};
 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct SparkTryParseUrl {
diff --git a/crates/sail-function/src/scalar/variant/spark_cast_to_variant.rs b/crates/sail-function/src/scalar/variant/spark_cast_to_variant.rs
index eeff9eb4d..fe28b8cc2 100644
--- a/crates/sail-function/src/scalar/variant/spark_cast_to_variant.rs
+++ b/crates/sail-function/src/scalar/variant/spark_cast_to_variant.rs
@@ -8,10 +8,10 @@ use datafusion::logical_expr::{
 };
 use datafusion::scalar::ScalarValue;
 use datafusion_common::exec_err;
-use parquet_variant_compute::{cast_to_variant, VariantType};
-use sail_common_datafusion::variant::variant_metadata_field;
+use parquet_variant_compute::{VariantType, cast_to_variant};
+use sail_common_datafusion::variant::{VARIANT_VALUE_FIELD_NAME, variant_metadata_field};
 
-use super::spark_parse_json::convert_binaryview_to_binary;
+use super::spark_parse_json::convert_variant_binaryview_to_binary;
 
 /// Implements `CAST(expr AS VARIANT)` for Spark-compatible variant conversion.
 ///
@@ -47,8 +47,8 @@ impl ScalarUDFImpl for SparkCastToVariant {
 
     fn return_type(&self, _arg_types: &[DataType]) -> Result {
         Ok(DataType::Struct(Fields::from(vec![
+            Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, false),
             variant_metadata_field(DataType::Binary, false),
-            Field::new("value", DataType::Binary, false),
         ])))
     }
 
@@ -86,8 +86,8 @@ impl ScalarUDFImpl for SparkCastToVariant {
                 if scalar.is_null() {
                     // SQL NULL → NULL Variant struct (must match promised return type)
                     let fields = Fields::from(vec![
+                        Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, false),
                         variant_metadata_field(DataType::Binary, false),
-                        Field::new("value", DataType::Binary, false),
                     ]);
                     let null_struct = StructArray::new_null(fields, 1);
                     return Ok(ColumnarValue::Scalar(ScalarValue::Struct(Arc::new(
@@ -97,7 +97,7 @@ impl ScalarUDFImpl for SparkCastToVariant {
                 let arr = scalar.to_array()?;
                 let variant_array = cast_to_variant(&arr)?;
                 let struct_array: StructArray = variant_array.into();
-                let struct_array = convert_binaryview_to_binary(struct_array)?;
+                let struct_array = convert_variant_binaryview_to_binary(struct_array)?;
                 Ok(ColumnarValue::Scalar(ScalarValue::Struct(Arc::new(
                     struct_array,
                 ))))
@@ -105,7 +105,7 @@ impl ScalarUDFImpl for SparkCastToVariant {
             ColumnarValue::Array(arr) => {
                 let variant_array = cast_to_variant(arr.as_ref())?;
                 let struct_array: StructArray = variant_array.into();
-                let struct_array = convert_binaryview_to_binary(struct_array)?;
+                let struct_array = convert_variant_binaryview_to_binary(struct_array)?;
                 Ok(ColumnarValue::Array(Arc::new(struct_array) as ArrayRef))
             }
         }
diff --git a/crates/sail-function/src/scalar/variant/spark_parse_json.rs b/crates/sail-function/src/scalar/variant/spark_parse_json.rs
index 05c209de0..18de71c72 100644
--- a/crates/sail-function/src/scalar/variant/spark_parse_json.rs
+++ b/crates/sail-function/src/scalar/variant/spark_parse_json.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 /// [Credit]: 
-use arrow::array::{new_null_array, Array, ArrayRef, StringViewArray, StructArray};
+use arrow::array::{Array, ArrayRef, StringViewArray, StructArray, new_null_array};
 use arrow::compute::cast;
 use arrow_schema::{DataType, Field, Fields};
 use datafusion::common::exec_datafusion_err;
@@ -13,8 +13,7 @@ use datafusion_expr_common::signature::Volatility;
 use parquet_variant_compute::{VariantArrayBuilder, VariantType};
 use parquet_variant_json::append_json;
 use sail_common_datafusion::variant::{
-    variant_metadata_field, VARIANT_METADATA_FIELD_NAME, VARIANT_METADATA_MARKER_KEY,
-    VARIANT_METADATA_MARKER_VALUE,
+    VARIANT_METADATA_FIELD_NAME, VARIANT_VALUE_FIELD_NAME, variant_metadata_field,
 };
 
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err};
@@ -188,8 +187,8 @@ impl ScalarUDFImpl for SparkParseJson {
         // internally and convert to Binary only at the Spark Connect serialization
         // layer, but that requires a broader refactor of the serialization path.
         Ok(DataType::Struct(Fields::from(vec![
+            Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, false),
             variant_metadata_field(DataType::Binary, false),
-            Field::new("value", DataType::Binary, false),
         ])))
     }
 
@@ -211,13 +210,14 @@ impl ScalarUDFImpl for SparkParseJson {
         // without parsing any rows. Placed after coerce_types has validated the
         // string arg type; the JSON parse itself is per-row, so there is no
         // batch-level validation that this short-circuit could silence.
-        if let Some(ColumnarValue::Array(arr)) = args.args.first() {
-            if !arr.is_empty() && arr.null_count() == arr.len() {
-                return Ok(ColumnarValue::Array(new_null_array(
-                    args.return_field.data_type(),
-                    arr.len(),
-                )));
-            }
+        if let Some(ColumnarValue::Array(arr)) = args.args.first()
+            && !arr.is_empty()
+            && arr.null_count() == arr.len()
+        {
+            return Ok(ColumnarValue::Array(new_null_array(
+                args.return_field.data_type(),
+                arr.len(),
+            )));
         }
 
         let safe = self.safe;
@@ -269,7 +269,7 @@ fn parse_json_kernel(args: &[ArrayRef], safe: bool, name: &str) -> Result Err(unsupported_data_type_exec_err(name, "string", other)),
@@ -305,7 +305,7 @@ pub(crate) fn from_utf8view_arr(arr: &ArrayRef, safe: bool) -> Result
     }
 
     let variant_array: StructArray = builder.build().into();
-    let variant_array = convert_binaryview_to_binary(variant_array)?;
+    let variant_array = convert_variant_binaryview_to_binary(variant_array)?;
     Ok(Arc::new(variant_array) as ArrayRef)
 }
 
@@ -315,17 +315,13 @@ pub(crate) fn convert_binaryview_to_binary(struct_array: StructArray) -> Result<
         .fields()
         .iter()
         .map(|f| {
-            let field = if f.name() == VARIANT_METADATA_FIELD_NAME {
-                let mut metadata = f.metadata().clone();
-                metadata.insert(
-                    VARIANT_METADATA_MARKER_KEY.to_string(),
-                    VARIANT_METADATA_MARKER_VALUE.to_string(),
-                );
-                variant_metadata_field(DataType::Binary, f.is_nullable()).with_metadata(metadata)
-            } else {
+            if matches!(f.data_type(), DataType::Binary) {
+                return f.clone();
+            }
+            Arc::new(
                 Field::new(f.name(), DataType::Binary, f.is_nullable())
-            };
-            Arc::new(field)
+                    .with_metadata(f.metadata().clone()),
+            )
         })
         .collect();
 
@@ -333,6 +329,9 @@ pub(crate) fn convert_binaryview_to_binary(struct_array: StructArray) -> Result<
         .columns()
         .iter()
         .map(|col| {
+            if matches!(col.data_type(), DataType::Binary) {
+                return Ok(col.clone());
+            }
             cast(col, &DataType::Binary)
                 .map_err(|e| exec_datafusion_err!("Failed to cast BinaryView to Binary: {e}"))
         })
@@ -345,15 +344,95 @@ pub(crate) fn convert_binaryview_to_binary(struct_array: StructArray) -> Result<
     ))
 }
 
+pub(crate) fn convert_variant_binaryview_to_binary(
+    struct_array: StructArray,
+) -> Result {
+    let struct_array = convert_binaryview_to_binary(struct_array)?;
+    let (value_index, value_field) = struct_array
+        .fields()
+        .find(VARIANT_VALUE_FIELD_NAME)
+        .ok_or_else(|| exec_datafusion_err!("missing variant field: {VARIANT_VALUE_FIELD_NAME}"))?;
+    let (metadata_index, metadata_field) = struct_array
+        .fields()
+        .find(VARIANT_METADATA_FIELD_NAME)
+        .ok_or_else(|| {
+            exec_datafusion_err!("missing variant field: {VARIANT_METADATA_FIELD_NAME}")
+        })?;
+    let value_column = struct_array.column(value_index);
+    let metadata_column = struct_array.column(metadata_index);
+
+    let field = variant_metadata_field(DataType::Binary, metadata_field.is_nullable());
+    let mut metadata = metadata_field.metadata().clone();
+    metadata.extend(field.metadata().clone());
+
+    Ok(StructArray::new(
+        Fields::from(vec![
+            value_field.clone(),
+            Arc::new(field.with_metadata(metadata)),
+        ]),
+        vec![value_column.clone(), metadata_column.clone()],
+        struct_array.nulls().cloned(),
+    ))
+}
+
 #[cfg(test)]
 mod tests {
     use datafusion::logical_expr::{ReturnFieldArgs, ScalarFunctionArgs};
-    use datafusion_common::{exec_err, ScalarValue};
+    use datafusion_common::{ScalarValue, exec_err};
     use parquet_variant::{Variant, VariantBuilder};
     use parquet_variant_compute::VariantArray;
+    use sail_common_datafusion::variant::is_variant_metadata_field;
 
     use super::*;
 
+    #[test]
+    fn test_variant_builder_output_is_marked_by_variant_binary_conversion() -> Result<()> {
+        let mut builder = VariantArrayBuilder::new(1);
+        builder.append_variant(Variant::from("x"));
+        let struct_array: StructArray = builder.build().into();
+
+        let names = struct_array
+            .fields()
+            .iter()
+            .map(|field| field.name().as_str())
+            .collect::>();
+        assert_eq!(
+            names,
+            vec![VARIANT_METADATA_FIELD_NAME, VARIANT_VALUE_FIELD_NAME]
+        );
+        assert!(!is_variant_metadata_field(
+            struct_array.fields()[0].as_ref()
+        ));
+
+        let generic_array = convert_binaryview_to_binary(struct_array.clone())?;
+        let names = generic_array
+            .fields()
+            .iter()
+            .map(|field| field.name().as_str())
+            .collect::>();
+        assert_eq!(
+            names,
+            vec![VARIANT_METADATA_FIELD_NAME, VARIANT_VALUE_FIELD_NAME]
+        );
+        assert!(!is_variant_metadata_field(
+            generic_array.fields()[0].as_ref()
+        ));
+
+        let struct_array = convert_variant_binaryview_to_binary(struct_array)?;
+        let names = struct_array
+            .fields()
+            .iter()
+            .map(|field| field.name().as_str())
+            .collect::>();
+        assert_eq!(
+            names,
+            vec![VARIANT_VALUE_FIELD_NAME, VARIANT_METADATA_FIELD_NAME]
+        );
+        assert!(is_variant_metadata_field(struct_array.fields()[1].as_ref()));
+
+        Ok(())
+    }
+
     #[test]
     fn test_json_to_variant_udf_scalar_none() -> Result<()> {
         let json_input = ScalarValue::Utf8(None);
diff --git a/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs b/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs
index dbe39b8bf..200b55760 100644
--- a/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs
+++ b/crates/sail-function/src/scalar/variant/spark_schema_of_variant.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use arrow::array::ArrayRef;
 use arrow_schema::DataType;
-use datafusion::common::{exec_datafusion_err, exec_err};
+use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err};
 use datafusion::error::Result;
 use datafusion::logical_expr::{
     ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
@@ -13,6 +13,7 @@ use parquet_variant_compute::VariantArray;
 
 use crate::error::invalid_arg_count_exec_err;
 use crate::scalar::variant::utils::helper::{try_field_as_variant_array, try_parse_variant_scalar};
+use crate::schema_inference::{InferredType, TypeMerger};
 
 /// Returns the schema (type string) of a variant value using Spark type names.
 ///
@@ -82,10 +83,8 @@ impl ScalarUDFImpl for SparkSchemaOfVariantUdf {
             }
             ColumnarValue::Array(variant_array) => {
                 let variant_array = VariantArray::try_new(variant_array.as_ref())?;
-                let mut builder = arrow::array::StringBuilder::with_capacity(
-                    variant_array.len(),
-                    variant_array.len() * 10,
-                );
+                let mut builder =
+                    arrow::array::StringBuilder::with_capacity(variant_array.len(), 0);
                 for v in variant_array.iter() {
                     match v {
                         Some(variant) => {
@@ -117,145 +116,163 @@ impl ScalarUDFImpl for SparkSchemaOfVariantUdf {
 
 /// Convert a Variant value to its Spark type string representation.
 pub(crate) fn variant_to_spark_type(variant: &Variant) -> String {
+    variant_type_to_spark_type(&variant_to_inferred_type(variant))
+}
+
+pub(crate) fn variant_to_inferred_type(variant: &Variant) -> InferredType {
     match variant {
-        Variant::Null => "VOID".to_string(),
-        Variant::BooleanTrue | Variant::BooleanFalse => "BOOLEAN".to_string(),
+        Variant::Null => InferredType::Null,
+        Variant::BooleanTrue | Variant::BooleanFalse => InferredType::Boolean,
         Variant::Int8(_) | Variant::Int16(_) | Variant::Int32(_) | Variant::Int64(_) => {
-            "BIGINT".to_string()
+            InferredType::Long
         }
-        Variant::Float(_) => "FLOAT".to_string(),
-        Variant::Double(_) => "DOUBLE".to_string(),
-        Variant::Decimal4(d) => decimal_type_string(d.integer() as i128, d.scale()),
-        Variant::Decimal8(d) => decimal_type_string(d.integer() as i128, d.scale()),
-        Variant::Decimal16(d) => decimal_type_string(d.integer(), d.scale()),
-        Variant::String(_) | Variant::ShortString(_) => "STRING".to_string(),
-        Variant::Binary(_) => "BINARY".to_string(),
-        Variant::Date(_) => "DATE".to_string(),
-        Variant::TimestampMicros(_) | Variant::TimestampNanos(_) => "TIMESTAMP".to_string(),
+        Variant::Float(_) => InferredType::Float,
+        Variant::Double(_) => InferredType::Double,
+        Variant::Decimal4(d) => decimal_type(d.integer() as i128, d.scale()),
+        Variant::Decimal8(d) => decimal_type(d.integer() as i128, d.scale()),
+        Variant::Decimal16(d) => decimal_type(d.integer(), d.scale()),
+        Variant::String(_) | Variant::ShortString(_) => InferredType::String,
+        Variant::Binary(_) => InferredType::Binary,
+        Variant::Date(_) => InferredType::Date,
+        Variant::TimestampMicros(_) | Variant::TimestampNanos(_) => InferredType::Timestamp,
         Variant::TimestampNtzMicros(_) | Variant::TimestampNtzNanos(_) => {
-            "TIMESTAMP_NTZ".to_string()
+            InferredType::TimestampNtz
         }
-        Variant::Time(_) => "STRING".to_string(),
-        Variant::Uuid(_) => "STRING".to_string(),
+        Variant::Time(_) | Variant::Uuid(_) => InferredType::String,
         Variant::Object(obj) => {
-            if obj.is_empty() {
-                return "OBJECT<>".to_string();
-            }
-            // Collect fields, sorted by field name (Spark sorts alphabetically)
-            let mut fields: Vec<(String, String)> = Vec::with_capacity(obj.len());
+            let mut fields = Vec::with_capacity(obj.len());
             for (name, value) in obj.iter() {
-                fields.push((name.to_string(), variant_to_spark_type(&value)));
+                fields.push((name.to_string(), variant_to_inferred_type(&value)));
             }
             fields.sort_by(|a, b| a.0.cmp(&b.0));
-            let fields_str: Vec = fields
-                .iter()
-                .map(|(name, ty)| format!("{name}: {ty}"))
-                .collect();
-            format!("OBJECT<{}>", fields_str.join(", "))
+            InferredType::Struct(fields)
         }
         Variant::List(list) => {
-            if list.is_empty() {
-                return "ARRAY".to_string();
+            let mut element_type = InferredType::Null;
+            for element in list.iter() {
+                element_type =
+                    merge_variant_types(element_type, variant_to_inferred_type(&element));
             }
-            let element_types: Vec =
-                list.iter().map(|e| variant_to_spark_type(&e)).collect();
-            let merged = merge_types(&element_types);
-            format!("ARRAY<{merged}>")
+            InferredType::Array(Box::new(element_type))
         }
     }
 }
 
-/// Merge a list of Spark type strings into a single type.
-///
-/// Rules (matching Spark behavior):
-/// - VOID is absorbed by any non-VOID type: `[VOID, BIGINT]` → `BIGINT`
-/// - OBJECT types with different fields are merged (field union): `[OBJECT, OBJECT]` → `OBJECT`
-/// - All other mismatches → `VARIANT`
-fn merge_types(types: &[String]) -> String {
-    // Filter out VOIDs
-    let non_void: Vec<&String> = types.iter().filter(|t| t.as_str() != "VOID").collect();
-
-    if non_void.is_empty() {
-        return "VOID".to_string();
-    }
+pub(crate) fn merge_variant_types(left: InferredType, right: InferredType) -> InferredType {
+    left.merge_with(right, &VariantTypeMerger)
+}
 
-    // Check if all non-void types are identical
-    if non_void.iter().all(|t| *t == non_void[0]) {
-        return non_void[0].clone();
-    }
+struct VariantTypeMerger;
 
-    // Try merging OBJECTs: if all non-void types are OBJECT<...>, union their fields
-    if non_void.iter().all(|t| t.starts_with("OBJECT<")) {
-        return merge_object_types(&non_void);
+impl TypeMerger for VariantTypeMerger {
+    fn merge_atomic(&self, _left: InferredType, _right: InferredType) -> InferredType {
+        InferredType::Variant
     }
-
-    "VARIANT".to_string()
 }
 
-/// Merge multiple OBJECT<...> type strings by unioning their fields.
-fn merge_object_types(types: &[&String]) -> String {
-    let mut merged_fields: std::collections::BTreeMap =
-        std::collections::BTreeMap::new();
-
-    for ty in types {
-        // Strip "OBJECT<" prefix and ">" suffix
-        let inner = ty
-            .strip_prefix("OBJECT<")
-            .and_then(|s| s.strip_suffix('>'))
-            .unwrap_or("");
-
-        if inner.is_empty() {
-            continue;
+pub(crate) fn variant_type_to_spark_type(inferred: &InferredType) -> String {
+    match inferred {
+        InferredType::Null => "VOID".to_string(),
+        InferredType::Boolean => "BOOLEAN".to_string(),
+        InferredType::Long => "BIGINT".to_string(),
+        InferredType::Float => "FLOAT".to_string(),
+        InferredType::Decimal(precision, scale) => format!("DECIMAL({precision},{scale})"),
+        InferredType::Double => "DOUBLE".to_string(),
+        InferredType::String => "STRING".to_string(),
+        InferredType::Binary => "BINARY".to_string(),
+        InferredType::Date => "DATE".to_string(),
+        InferredType::Timestamp => "TIMESTAMP".to_string(),
+        InferredType::TimestampNtz => "TIMESTAMP_NTZ".to_string(),
+        InferredType::Array(element) => {
+            format!("ARRAY<{}>", variant_type_to_spark_type(element))
         }
-
-        // Parse fields: "a: BIGINT, b: STRING" → [("a", "BIGINT"), ("b", "STRING")]
-        // Handle nested types by tracking angle bracket depth
-        for field_str in split_fields(inner) {
-            if let Some((name, field_type)) = field_str.split_once(':') {
-                let name = name.trim().to_string();
-                let field_type = field_type.trim().to_string();
-                merged_fields.entry(name).or_insert(field_type);
-            }
+        InferredType::Struct(fields) => {
+            let fields = fields
+                .iter()
+                .map(|(name, ty)| format!("{name}: {}", variant_type_to_spark_type(ty)))
+                .collect::>();
+            format!("OBJECT<{}>", fields.join(", "))
         }
+        InferredType::Variant => "VARIANT".to_string(),
     }
+}
 
-    if merged_fields.is_empty() {
-        return "OBJECT<>".to_string();
+pub(crate) fn variant_type_from_spark_type(s: &str) -> Result {
+    match s {
+        "VOID" => return Ok(InferredType::Null),
+        "BOOLEAN" => return Ok(InferredType::Boolean),
+        "BIGINT" => return Ok(InferredType::Long),
+        "FLOAT" => return Ok(InferredType::Float),
+        "DOUBLE" => return Ok(InferredType::Double),
+        "STRING" => return Ok(InferredType::String),
+        "BINARY" => return Ok(InferredType::Binary),
+        "DATE" => return Ok(InferredType::Date),
+        "TIMESTAMP" => return Ok(InferredType::Timestamp),
+        "TIMESTAMP_NTZ" => return Ok(InferredType::TimestampNtz),
+        "VARIANT" => return Ok(InferredType::Variant),
+        _ => {}
     }
-
-    let fields_str: Vec = merged_fields
-        .iter()
-        .map(|(name, ty)| format!("{name}: {ty}"))
-        .collect();
-    format!("OBJECT<{}>", fields_str.join(", "))
+    if let Some(inner) = s.strip_prefix("DECIMAL(").and_then(|s| s.strip_suffix(')')) {
+        let (precision, scale) = inner.split_once(',').ok_or_else(|| {
+            DataFusionError::Execution(format!("invalid inferred decimal type '{s}'"))
+        })?;
+        let precision = precision.parse::().map_err(|_| {
+            DataFusionError::Execution(format!("invalid inferred decimal precision '{precision}'"))
+        })?;
+        let scale = scale.parse::().map_err(|_| {
+            DataFusionError::Execution(format!("invalid inferred decimal scale '{scale}'"))
+        })?;
+        return Ok(InferredType::Decimal(precision, scale));
+    }
+    if let Some(inner) = s.strip_prefix("ARRAY<").and_then(|s| s.strip_suffix('>')) {
+        return Ok(InferredType::Array(Box::new(variant_type_from_spark_type(
+            inner,
+        )?)));
+    }
+    if let Some(inner) = s.strip_prefix("OBJECT<").and_then(|s| s.strip_suffix('>')) {
+        let mut fields = Vec::new();
+        for field in split_top_level(inner) {
+            let (name, ty) = field.split_once(": ").ok_or_else(|| {
+                DataFusionError::Execution(format!("invalid inferred object field '{field}'"))
+            })?;
+            fields.push((
+                name.trim().to_string(),
+                variant_type_from_spark_type(ty.trim())?,
+            ));
+        }
+        return Ok(InferredType::Struct(fields));
+    }
+    Err(DataFusionError::Execution(format!(
+        "invalid inferred variant type '{s}'"
+    )))
 }
 
-/// Split a comma-separated field list respecting nested angle brackets.
-/// e.g. "a: OBJECT, b: ARRAY" → ["a: OBJECT", "b: ARRAY"]
-fn split_fields(s: &str) -> Vec<&str> {
-    let mut result = Vec::new();
-    let mut depth = 0;
+fn split_top_level(s: &str) -> Vec<&str> {
+    let mut fields = Vec::new();
+    let mut angle_depth = 0usize;
+    let mut paren_depth = 0usize;
     let mut start = 0;
     for (i, ch) in s.char_indices() {
         match ch {
-            '<' => depth += 1,
-            '>' => depth -= 1,
-            ',' if depth == 0 => {
-                result.push(s[start..i].trim());
+            '<' => angle_depth += 1,
+            '>' => angle_depth = angle_depth.saturating_sub(1),
+            '(' => paren_depth += 1,
+            ')' => paren_depth = paren_depth.saturating_sub(1),
+            ',' if angle_depth == 0 && paren_depth == 0 => {
+                fields.push(s[start..i].trim());
                 start = i + 1;
             }
             _ => {}
         }
     }
-    let last = s[start..].trim();
-    if !last.is_empty() {
-        result.push(last);
+    if start < s.len() {
+        fields.push(s[start..].trim());
     }
-    result
+    fields
 }
 
 /// Compute the decimal precision from the integer value and scale.
-fn decimal_type_string(integer: i128, scale: u8) -> String {
+fn decimal_type(integer: i128, scale: u8) -> InferredType {
     let abs = integer.unsigned_abs();
     let int_digits = if abs == 0 {
         1u8
@@ -270,5 +287,5 @@ fn decimal_type_string(integer: i128, scale: u8) -> String {
         d
     };
     let precision = int_digits.max(scale);
-    format!("DECIMAL({precision},{scale})")
+    InferredType::Decimal(precision, scale)
 }
diff --git a/crates/sail-function/src/scalar/variant/spark_to_variant_object.rs b/crates/sail-function/src/scalar/variant/spark_to_variant_object.rs
index 54a46e5a5..2b3507ee5 100644
--- a/crates/sail-function/src/scalar/variant/spark_to_variant_object.rs
+++ b/crates/sail-function/src/scalar/variant/spark_to_variant_object.rs
@@ -8,11 +8,11 @@ use datafusion::logical_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
 use datafusion::scalar::ScalarValue;
-use parquet_variant_compute::{cast_to_variant, VariantType};
-use sail_common_datafusion::variant::variant_metadata_field;
+use parquet_variant_compute::{VariantType, cast_to_variant};
+use sail_common_datafusion::variant::{VARIANT_VALUE_FIELD_NAME, variant_metadata_field};
 
 use crate::error::{invalid_arg_count_exec_err, unsupported_data_type_exec_err};
-use crate::scalar::variant::spark_parse_json::convert_binaryview_to_binary;
+use crate::scalar::variant::spark_parse_json::convert_variant_binaryview_to_binary;
 
 /// Recursively checks if a DataType contains Null (VOID) anywhere.
 fn contains_void_type(dt: &DataType) -> bool {
@@ -74,8 +74,8 @@ impl ScalarUDFImpl for SparkToVariantObjectUdf {
 
     fn return_type(&self, _arg_types: &[DataType]) -> Result {
         Ok(DataType::Struct(Fields::from(vec![
+            Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, false),
             variant_metadata_field(DataType::Binary, false),
-            Field::new("value", DataType::Binary, false),
         ])))
     }
 
@@ -126,8 +126,8 @@ impl ScalarUDFImpl for SparkToVariantObjectUdf {
             ColumnarValue::Scalar(scalar) => {
                 if scalar.is_null() {
                     let fields = Fields::from(vec![
+                        Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, false),
                         variant_metadata_field(DataType::Binary, false),
-                        Field::new("value", DataType::Binary, false),
                     ]);
                     let null_struct = StructArray::new_null(fields, 1);
                     return Ok(ColumnarValue::Scalar(ScalarValue::Struct(Arc::new(
@@ -139,7 +139,7 @@ impl ScalarUDFImpl for SparkToVariantObjectUdf {
                     exec_datafusion_err!("to_variant_object: failed to convert to VARIANT: {e}")
                 })?;
                 let struct_array: StructArray = variant_array.into();
-                let struct_array = convert_binaryview_to_binary(struct_array)?;
+                let struct_array = convert_variant_binaryview_to_binary(struct_array)?;
                 Ok(ColumnarValue::Scalar(ScalarValue::Struct(Arc::new(
                     struct_array,
                 ))))
@@ -149,7 +149,7 @@ impl ScalarUDFImpl for SparkToVariantObjectUdf {
                     exec_datafusion_err!("to_variant_object: failed to convert to VARIANT: {e}")
                 })?;
                 let struct_array: StructArray = variant_array.into();
-                let struct_array = convert_binaryview_to_binary(struct_array)?;
+                let struct_array = convert_variant_binaryview_to_binary(struct_array)?;
                 Ok(ColumnarValue::Array(Arc::new(struct_array) as ArrayRef))
             }
         }
diff --git a/crates/sail-function/src/scalar/variant/spark_variant_explode.rs b/crates/sail-function/src/scalar/variant/spark_variant_explode.rs
index 31edf9be7..cbd1c961c 100644
--- a/crates/sail-function/src/scalar/variant/spark_variant_explode.rs
+++ b/crates/sail-function/src/scalar/variant/spark_variant_explode.rs
@@ -11,18 +11,18 @@ use datafusion::logical_expr::{
 use datafusion::scalar::ScalarValue;
 use parquet_variant::Variant;
 use parquet_variant_compute::{VariantArray, VariantArrayBuilder, VariantType};
-use sail_common_datafusion::variant::variant_metadata_field;
+use sail_common_datafusion::variant::{VARIANT_VALUE_FIELD_NAME, variant_metadata_field};
 
 use crate::error::invalid_arg_count_exec_err;
-use crate::scalar::variant::spark_parse_json::convert_binaryview_to_binary;
+use crate::scalar::variant::spark_parse_json::convert_variant_binaryview_to_binary;
 use crate::scalar::variant::utils::helper::try_field_as_variant_array;
 
 fn variant_explode_value_field() -> Field {
     Field::new(
-        "value",
+        VARIANT_VALUE_FIELD_NAME,
         DataType::Struct(Fields::from(vec![
+            Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, false),
             variant_metadata_field(DataType::Binary, false),
-            Field::new("value", DataType::Binary, false),
         ])),
         true,
     )
@@ -223,7 +223,7 @@ fn build_list_array(
     let key_arr: ArrayRef = Arc::new(key_builder.finish());
 
     let value_struct: StructArray = value_builder.build().into();
-    let value_struct = convert_binaryview_to_binary(value_struct)?;
+    let value_struct = convert_variant_binaryview_to_binary(value_struct)?;
     let value_arr: ArrayRef = Arc::new(value_struct);
 
     let struct_arr = StructArray::try_new(
diff --git a/crates/sail-function/src/scalar/variant/spark_variant_get.rs b/crates/sail-function/src/scalar/variant/spark_variant_get.rs
index 20645b8cd..521d49fe2 100644
--- a/crates/sail-function/src/scalar/variant/spark_variant_get.rs
+++ b/crates/sail-function/src/scalar/variant/spark_variant_get.rs
@@ -4,13 +4,13 @@ use std::sync::Arc;
 use arrow_schema::{ArrowError, DataType, Field, FieldRef};
 use chumsky::prelude::*;
 use datafusion::arrow::datatypes::TimeUnit;
-use datafusion_common::{arrow_datafusion_err, exec_datafusion_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err};
 use datafusion_expr::{
     ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
 use parquet_variant::{VariantPath, VariantPathElement};
-use parquet_variant_compute::{variant_get, GetOptions, VariantType};
-use sail_common_datafusion::variant::variant_metadata_field;
+use parquet_variant_compute::{GetOptions, VariantType, variant_get};
+use sail_common_datafusion::variant::{VARIANT_VALUE_FIELD_NAME, variant_metadata_field};
 
 use crate::error::{generic_exec_err, invalid_arg_count_exec_err, unsupported_data_type_exec_err};
 use crate::scalar::variant::utils::helper::{try_field_as_variant_array, try_parse_string_scalar};
@@ -52,12 +52,11 @@ fn build_get_options<'a>(path: VariantPath<'a>, as_type: &Option) -> G
 /// - 2-arg form: returns a Variant struct (Binary for PySpark compat).
 fn return_field_for_variant_get(name: &str, args: &ReturnFieldArgs) -> Result {
     // Validate path (arg 1) is a constant non-null string
-    if let Some(path_opt) = args.scalar_arguments.get(1) {
-        if let Some(sv) = path_opt.as_ref() {
-            if sv.try_as_str().flatten().is_none() {
-                return Err(generic_exec_err(name, "path must be a non-null string"));
-            }
-        }
+    if let Some(path_opt) = args.scalar_arguments.get(1)
+        && let Some(sv) = path_opt.as_ref()
+        && sv.try_as_str().flatten().is_none()
+    {
+        return Err(generic_exec_err(name, "path must be a non-null string"));
     }
 
     // 3-arg form: variant_get(variant, path, type) → typed result
@@ -75,8 +74,8 @@ fn return_field_for_variant_get(name: &str, args: &ReturnFieldArgs) -> Result Resul
             &result,
             &DataType::Struct(
                 vec![
+                    Field::new(VARIANT_VALUE_FIELD_NAME, DataType::Binary, true),
                     variant_metadata_field(DataType::Binary, false),
-                    Field::new("value", DataType::Binary, true),
                 ]
                 .into(),
             ),
@@ -304,7 +303,7 @@ impl ScalarUDFImpl for SparkVariantGet {
                     self.name(),
                     "string",
                     &arg_types[1],
-                ))
+                ));
             }
         });
         if arg_types.len() == 3 {
@@ -316,7 +315,7 @@ impl ScalarUDFImpl for SparkVariantGet {
                         self.name(),
                         "string",
                         &arg_types[2],
-                    ))
+                    ));
                 }
             });
         }
@@ -379,7 +378,7 @@ fn spark_type_to_arrow(type_str: &str) -> Result {
         "binary" => return Ok(DataType::Binary),
         "date" => return Ok(DataType::Date32),
         "timestamp" | "timestamp_ntz" => {
-            return Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
+            return Ok(DataType::Timestamp(TimeUnit::Microsecond, None));
         }
         _ => {}
     }
@@ -422,8 +421,8 @@ fn quoted_field_name<'src>(
         .then_ignore(just(quote))
 }
 
-fn spark_variant_path_parser<'src>(
-) -> impl Parser<'src, &'src str, VariantPath<'static>, extra::Err>> {
+fn spark_variant_path_parser<'src>()
+-> impl Parser<'src, &'src str, VariantPath<'static>, extra::Err>> {
     let ident_field_name = text::ident().map(|s: &str| VariantPathElement::field(s.to_string()));
 
     let single_quoted_field_name = quoted_field_name('\'').map(VariantPathElement::field);
diff --git a/crates/sail-function/src/scalar/variant/spark_variant_to_json.rs b/crates/sail-function/src/scalar/variant/spark_variant_to_json.rs
index 47d56666a..d446340ba 100644
--- a/crates/sail-function/src/scalar/variant/spark_variant_to_json.rs
+++ b/crates/sail-function/src/scalar/variant/spark_variant_to_json.rs
@@ -23,7 +23,9 @@ pub fn variant_to_json_columnar(arg: &ColumnarValue) -> Result {
             ScalarValue::Struct(variant_array) => {
                 let variant_array = VariantArray::try_new(variant_array.as_ref())?;
                 if variant_array.is_empty() {
-                    return exec_err!("Cannot convert empty VariantArray to JSON: the array must contain at least one element");
+                    return exec_err!(
+                        "Cannot convert empty VariantArray to JSON: the array must contain at least one element"
+                    );
                 }
                 if variant_array.is_null(0) {
                     Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(None)))
diff --git a/crates/sail-function/src/scalar/variant/utils/helper.rs b/crates/sail-function/src/scalar/variant/utils/helper.rs
index eaf581489..40f488b72 100644
--- a/crates/sail-function/src/scalar/variant/utils/helper.rs
+++ b/crates/sail-function/src/scalar/variant/utils/helper.rs
@@ -1,7 +1,7 @@
 /// [Credit]: 
 use arrow_schema::extension::ExtensionType;
 use arrow_schema::{DataType, Field};
-use datafusion_common::{exec_err, ScalarValue};
+use datafusion_common::{ScalarValue, exec_err};
 use parquet_variant_compute::{VariantArray, VariantType};
 use sail_common_datafusion::variant::{is_variant_arrow_field, is_variant_storage_field};
 
diff --git a/crates/sail-function/src/scalar/xml/from_xml.rs b/crates/sail-function/src/scalar/xml/from_xml.rs
new file mode 100644
index 000000000..4df75855a
--- /dev/null
+++ b/crates/sail-function/src/scalar/xml/from_xml.rs
@@ -0,0 +1,1266 @@
+use std::sync::Arc;
+
+use datafusion::arrow::array::timezone::Tz;
+use datafusion::arrow::array::*;
+use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
+use datafusion::arrow::datatypes::*;
+use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+};
+use datafusion_expr_common::signature::Volatility;
+use sail_common::spec::{
+    self, SAIL_LIST_FIELD_NAME, SAIL_MAP_FIELD_NAME, SAIL_MAP_KEY_FIELD_NAME,
+    SAIL_MAP_VALUE_FIELD_NAME,
+};
+use sail_common_datafusion::utils::datetime::localize_with_fallback;
+use sail_sql_analyzer::data_type::from_ast_data_type;
+use sail_sql_analyzer::parser as sail_parser;
+use xee_xpath::Documents;
+
+use crate::functions_utils::make_scalar_function;
+use crate::scalar::datetime::format::{DateTimeFormat, ParsedDateTime};
+
+#[cfg(test)]
+const DEFAULT_SESSION_TIMEZONE: &str = "UTC";
+
+/// UDF implementation of `from_xml`, similar to Spark's `XmlToStructs`
+/// Parses an XML string column into a struct column using a user-provided schema
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkFromXml {
+    session_timezone: Arc,
+    signature: Signature,
+}
+
+impl SparkFromXml {
+    pub const FROM_XML_NAME: &'static str = "from_xml";
+
+    pub fn new(session_timezone: Arc) -> Self {
+        Self {
+            session_timezone,
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+
+    pub fn session_timezone(&self) -> &str {
+        &self.session_timezone
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ParseMode {
+    Permissive,
+    FailFast,
+}
+
+#[derive(Debug)]
+struct SparkFromXmlOptions {
+    null_value: Option,
+    attribute_prefix: String,
+    value_tag: String,
+    timestamp_ltz_format: DateTimeFormat,
+    timestamp_ntz_format: DateTimeFormat,
+    date_format: DateTimeFormat,
+    mode: ParseMode,
+}
+
+impl SparkFromXmlOptions {
+    const NULL_VALUE_OPTION: &'static str = "nullValue";
+    const ATTRIBUTE_PREFIX_OPTION: &'static str = "attributePrefix";
+    const ATTRIBUTE_PREFIX_DEFAULT: &'static str = "_";
+    const VALUE_TAG_OPTION: &'static str = "valueTag";
+    const VALUE_TAG_DEFAULT: &'static str = "_VALUE";
+    const TIMESTAMP_FORMAT_OPTION: &'static str = "timestampFormat";
+    const TIMESTAMP_NTZ_FORMAT_OPTION: &'static str = "timestampNTZFormat";
+    const DATE_FORMAT_OPTION: &'static str = "dateFormat";
+    const MODE_OPTION: &'static str = "mode";
+    const TIMESTAMP_LTZ_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+    const TIMESTAMP_NTZ_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+    const DATE_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd";
+
+    fn from_map(map: &MapArray) -> Result {
+        let null_value = find_key_value(map, Self::NULL_VALUE_OPTION);
+        let attribute_prefix = find_key_value(map, Self::ATTRIBUTE_PREFIX_OPTION)
+            .unwrap_or_else(|| Self::ATTRIBUTE_PREFIX_DEFAULT.to_string());
+        let value_tag = find_key_value(map, Self::VALUE_TAG_OPTION)
+            .unwrap_or_else(|| Self::VALUE_TAG_DEFAULT.to_string());
+        if value_tag.is_empty() {
+            return plan_err!("`valueTag` must not be empty");
+        }
+        if value_tag == attribute_prefix {
+            return plan_err!("`valueTag` and `attributePrefix` must not be equal");
+        }
+        let timestamp_ltz_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
+            .as_deref()
+            .map(DateTimeFormat::for_parsing)
+            .transpose()?
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_LTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp LTZ format should be valid")
+            });
+        let timestamp_ntz_format = find_key_value(map, Self::TIMESTAMP_NTZ_FORMAT_OPTION)
+            .as_deref()
+            .map(DateTimeFormat::for_parsing)
+            .transpose()?
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp NTZ format should be valid")
+            });
+        let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
+            .as_deref()
+            .map(DateTimeFormat::for_parsing)
+            .transpose()?
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            });
+        let mode = match find_key_value(map, Self::MODE_OPTION)
+            .as_deref()
+            .map(|s| s.to_ascii_uppercase())
+            .as_deref()
+        {
+            None | Some("PERMISSIVE") => ParseMode::Permissive,
+            Some("FAILFAST") => ParseMode::FailFast,
+            Some(other) => {
+                return plan_err!("`mode` must be PERMISSIVE or FAILFAST, got '{other}'");
+            }
+        };
+        Ok(Self {
+            null_value,
+            attribute_prefix,
+            value_tag,
+            timestamp_ltz_format,
+            timestamp_ntz_format,
+            date_format,
+            mode,
+        })
+    }
+}
+
+impl Default for SparkFromXmlOptions {
+    fn default() -> Self {
+        Self {
+            null_value: None,
+            attribute_prefix: Self::ATTRIBUTE_PREFIX_DEFAULT.to_string(),
+            value_tag: Self::VALUE_TAG_DEFAULT.to_string(),
+            timestamp_ltz_format: {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_LTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp LTZ format should be valid")
+            },
+            timestamp_ntz_format: {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp NTZ format should be valid")
+            },
+            date_format: {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_parsing(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            },
+            mode: ParseMode::Permissive,
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkFromXml {
+    fn name(&self) -> &str {
+        Self::FROM_XML_NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result {
+        Ok(DataType::Struct(Fields::empty()))
+    }
+
+    fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result {
+        let schema_str = match args.scalar_arguments.get(1) {
+            Some(Some(
+                ScalarValue::Utf8(Some(s))
+                | ScalarValue::LargeUtf8(Some(s))
+                | ScalarValue::Utf8View(Some(s)),
+            )) => s.as_str(),
+            _ => {
+                return plan_err!(
+                    "`{}` requires the schema argument to be a string literal",
+                    Self::FROM_XML_NAME
+                );
+            }
+        };
+        let dt = parse_xml_schema(schema_str, &self.session_timezone)?;
+        Ok(Arc::new(Field::new(self.name(), dt, true)))
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        match arg_types {
+            [
+                DataType::Null | DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+            ] => Ok(vec![DataType::Utf8, DataType::Utf8]),
+            [
+                DataType::Null | DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8,
+                DataType::Map(_, _),
+            ] => Ok(vec![DataType::Utf8, DataType::Utf8, arg_types[2].clone()]),
+            _ => plan_err!(
+                "`{}` requires 2 or 3 arguments: xml STRING, schema STRING, options MAP (optional), got {:?}",
+                Self::FROM_XML_NAME,
+                arg_types
+            ),
+        }
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        let session_timezone = self.session_timezone.to_string();
+        let ScalarFunctionArgs { args, .. } = args;
+        make_scalar_function(
+            move |inner_args| spark_from_xml_inner(inner_args, session_timezone.as_str()),
+            vec![],
+        )(&args)
+    }
+}
+
+fn spark_from_xml_inner(args: &[ArrayRef], session_timezone: &str) -> Result {
+    if args.len() < 2 || args.len() > 3 {
+        return exec_err!(
+            "`{}` requires 2 or 3 arguments, got {}",
+            SparkFromXml::FROM_XML_NAME,
+            args.len()
+        );
+    }
+
+    let xml_array = args[0]
+        .as_any()
+        .downcast_ref::()
+        .ok_or_else(|| {
+            DataFusionError::Internal("from_xml: expected StringArray for arg 0".to_string())
+        })?;
+
+    let schema_array = args[1]
+        .as_any()
+        .downcast_ref::()
+        .ok_or_else(|| {
+            DataFusionError::Internal("from_xml: expected StringArray for arg 1".to_string())
+        })?;
+
+    if schema_array.is_empty() || schema_array.is_null(0) {
+        return exec_err!(
+            "`{}` requires a non-empty schema string",
+            SparkFromXml::FROM_XML_NAME
+        );
+    }
+    let schema_str = schema_array.value(0);
+
+    let options = if let Some(opts) = args.get(2) {
+        let map = opts.as_any().downcast_ref::().ok_or_else(|| {
+            DataFusionError::Internal("from_xml: expected MapArray for arg 2".to_string())
+        })?;
+        SparkFromXmlOptions::from_map(map)?
+    } else {
+        SparkFromXmlOptions::default()
+    };
+
+    let schema_dt = parse_xml_schema(schema_str, session_timezone)?;
+    let DataType::Struct(fields) = &schema_dt else {
+        return exec_err!(
+            "`{}` schema must resolve to a STRUCT type, got {:?}",
+            SparkFromXml::FROM_XML_NAME,
+            schema_dt
+        );
+    };
+
+    let mut builder = create_struct_builder(fields, xml_array.len(), session_timezone)?;
+
+    for i in 0..xml_array.len() {
+        if xml_array.is_null(i) {
+            append_null_struct(&mut builder)?;
+            continue;
+        }
+
+        let xml_str = xml_array.value(i);
+        match parse_xml_into_builder(xml_str, &mut builder, &options, session_timezone) {
+            Ok(()) => {}
+            Err(e) => {
+                if options.mode == ParseMode::FailFast {
+                    return Err(e);
+                }
+                append_null_children_struct(&mut builder)?;
+            }
+        }
+    }
+
+    finish_struct_builder(builder)
+}
+
+fn parse_xml_schema(schema: &str, session_timezone: &str) -> Result {
+    let schema = schema.trim();
+    let ast_result = sail_parser::parse_data_type(schema)
+        .or_else(|_| sail_parser::parse_data_type(&format!("STRUCT<{schema}>")));
+    let ast = ast_result
+        .map_err(|e| DataFusionError::Plan(format!("Failed to parse schema '{schema}': {e}")))?;
+    let spec_dt = from_ast_data_type(ast)
+        .map_err(|e| DataFusionError::Plan(format!("Failed to analyze schema '{schema}': {e}")))?;
+    spec_to_arrow_data_type(&spec_dt, session_timezone)
+}
+
+fn spec_to_arrow_data_type(dt: &spec::DataType, session_timezone: &str) -> Result {
+    use spec::DataType as SDT;
+
+    fn to_time_unit(unit: &spec::TimeUnit) -> TimeUnit {
+        match unit {
+            spec::TimeUnit::Second => TimeUnit::Second,
+            spec::TimeUnit::Millisecond => TimeUnit::Millisecond,
+            spec::TimeUnit::Microsecond => TimeUnit::Microsecond,
+            spec::TimeUnit::Nanosecond => TimeUnit::Nanosecond,
+        }
+    }
+
+    match dt {
+        SDT::Null => Ok(DataType::Null),
+        SDT::Boolean => Ok(DataType::Boolean),
+        SDT::Int8 => Ok(DataType::Int8),
+        SDT::Int16 => Ok(DataType::Int16),
+        SDT::Int32 => Ok(DataType::Int32),
+        SDT::Int64 => Ok(DataType::Int64),
+        SDT::UInt8 => Ok(DataType::UInt8),
+        SDT::UInt16 => Ok(DataType::UInt16),
+        SDT::UInt32 => Ok(DataType::UInt32),
+        SDT::UInt64 => Ok(DataType::UInt64),
+        SDT::Float16 => Ok(DataType::Float16),
+        SDT::Float32 => Ok(DataType::Float32),
+        SDT::Float64 => Ok(DataType::Float64),
+        SDT::Binary | SDT::ConfiguredBinary => Ok(DataType::Binary),
+        SDT::FixedSizeBinary { size } => Ok(DataType::FixedSizeBinary(*size)),
+        SDT::LargeBinary => Ok(DataType::LargeBinary),
+        SDT::BinaryView => Ok(DataType::BinaryView),
+        SDT::Utf8 | SDT::ConfiguredUtf8 { .. } => Ok(DataType::Utf8),
+        SDT::LargeUtf8 => Ok(DataType::LargeUtf8),
+        SDT::Utf8View => Ok(DataType::Utf8View),
+        SDT::Date32 => Ok(DataType::Date32),
+        SDT::Date64 => Ok(DataType::Date64),
+        SDT::Timestamp {
+            time_unit,
+            timestamp_type,
+        } => Ok(DataType::Timestamp(
+            to_time_unit(time_unit),
+            match timestamp_type {
+                spec::TimestampType::Configured | spec::TimestampType::WithLocalTimeZone => {
+                    Some(Arc::from(session_timezone))
+                }
+                spec::TimestampType::WithoutTimeZone => None,
+            },
+        )),
+        SDT::Time32 { time_unit: u } => Ok(DataType::Time32(to_time_unit(u))),
+        SDT::Time64 { time_unit: u } => Ok(DataType::Time64(to_time_unit(u))),
+        SDT::Duration { time_unit: u } => Ok(DataType::Duration(to_time_unit(u))),
+        SDT::Interval { interval_unit, .. } => match interval_unit {
+            spec::IntervalUnit::YearMonth => Ok(DataType::Interval(IntervalUnit::YearMonth)),
+            spec::IntervalUnit::DayTime => Ok(DataType::Duration(TimeUnit::Microsecond)),
+            spec::IntervalUnit::MonthDayNano => Ok(DataType::Interval(IntervalUnit::MonthDayNano)),
+        },
+        SDT::Decimal128 { precision, scale } => Ok(DataType::Decimal128(*precision, *scale)),
+        SDT::Decimal256 { precision, scale } => Ok(DataType::Decimal256(*precision, *scale)),
+        SDT::List {
+            data_type,
+            nullable,
+        } => Ok(DataType::List(Arc::new(Field::new(
+            SAIL_LIST_FIELD_NAME,
+            spec_to_arrow_data_type(data_type.as_ref(), session_timezone)?,
+            *nullable,
+        )))),
+        SDT::FixedSizeList {
+            data_type,
+            nullable,
+            length,
+        } => Ok(DataType::FixedSizeList(
+            Arc::new(Field::new(
+                SAIL_LIST_FIELD_NAME,
+                spec_to_arrow_data_type(data_type.as_ref(), session_timezone)?,
+                *nullable,
+            )),
+            *length,
+        )),
+        SDT::LargeList {
+            data_type,
+            nullable,
+        } => Ok(DataType::LargeList(Arc::new(Field::new(
+            SAIL_LIST_FIELD_NAME,
+            spec_to_arrow_data_type(data_type.as_ref(), session_timezone)?,
+            *nullable,
+        )))),
+        SDT::Struct { fields } => {
+            let mut out: Vec> = Vec::with_capacity(fields.len());
+            for f in fields.iter() {
+                out.push(Arc::new(Field::new(
+                    f.name.clone(),
+                    spec_to_arrow_data_type(&f.data_type, session_timezone)?,
+                    f.nullable,
+                )));
+            }
+            Ok(DataType::Struct(Fields::from(out)))
+        }
+        SDT::Map {
+            key_type,
+            value_type,
+            value_type_nullable,
+            keys_sorted,
+        } => {
+            let fields = Fields::from(vec![
+                Arc::new(Field::new(
+                    SAIL_MAP_KEY_FIELD_NAME,
+                    spec_to_arrow_data_type(key_type.as_ref(), session_timezone)?,
+                    false,
+                )),
+                Arc::new(Field::new(
+                    SAIL_MAP_VALUE_FIELD_NAME,
+                    spec_to_arrow_data_type(value_type.as_ref(), session_timezone)?,
+                    *value_type_nullable,
+                )),
+            ]);
+            Ok(DataType::Map(
+                Arc::new(Field::new(
+                    SAIL_MAP_FIELD_NAME,
+                    DataType::Struct(fields),
+                    false,
+                )),
+                *keys_sorted,
+            ))
+        }
+        SDT::Geometry { .. } | SDT::Geography { .. } => Ok(DataType::Binary),
+        SDT::Variant => Ok(DataType::Struct(Fields::from(vec![
+            Arc::new(Field::new("metadata", DataType::Binary, false)),
+            Arc::new(Field::new("value", DataType::Binary, false)),
+        ]))),
+        SDT::UserDefined { sql_type, .. } => {
+            spec_to_arrow_data_type(sql_type.as_ref(), session_timezone)
+        }
+        other => Err(DataFusionError::Plan(format!(
+            "Unsupported data type in from_xml schema: {other:?}"
+        ))),
+    }
+}
+
+enum XmlFieldBuilder {
+    Boolean(BooleanBuilder),
+    Int8(Int8Builder),
+    Int16(Int16Builder),
+    Int32(Int32Builder),
+    Int64(Int64Builder),
+    Float32(Float32Builder),
+    Float64(Float64Builder),
+    Decimal128 {
+        builder: Decimal128Builder,
+        precision: u8,
+        scale: i8,
+    },
+    String(StringBuilder),
+    Date32(Date32Builder),
+    TimestampMicrosecond {
+        builder: TimestampMicrosecondBuilder,
+        has_tz: bool,
+    },
+    TimestampNanosecond {
+        builder: TimestampNanosecondBuilder,
+        has_tz: bool,
+    },
+    List {
+        field: FieldRef,
+        offsets: Vec,
+        values: Box,
+        nulls: Vec,
+    },
+    Struct {
+        fields: Fields,
+        children: Vec,
+        nulls: Vec,
+    },
+    Unsupported {
+        data_type: DataType,
+        count: usize,
+    },
+}
+
+struct TopLevelStructBuilder {
+    fields: Fields,
+    children: Vec,
+    nulls: Vec,
+}
+
+#[expect(clippy::only_used_in_recursion)]
+fn create_field_builder(
+    data_type: &DataType,
+    capacity: usize,
+    session_timezone: &str,
+) -> Result {
+    match data_type {
+        DataType::Boolean => Ok(XmlFieldBuilder::Boolean(BooleanBuilder::with_capacity(
+            capacity,
+        ))),
+        DataType::Int8 => Ok(XmlFieldBuilder::Int8(Int8Builder::with_capacity(capacity))),
+        DataType::Int16 => Ok(XmlFieldBuilder::Int16(Int16Builder::with_capacity(
+            capacity,
+        ))),
+        DataType::Int32 => Ok(XmlFieldBuilder::Int32(Int32Builder::with_capacity(
+            capacity,
+        ))),
+        DataType::Int64 => Ok(XmlFieldBuilder::Int64(Int64Builder::with_capacity(
+            capacity,
+        ))),
+        DataType::Float32 => Ok(XmlFieldBuilder::Float32(Float32Builder::with_capacity(
+            capacity,
+        ))),
+        DataType::Float64 => Ok(XmlFieldBuilder::Float64(Float64Builder::with_capacity(
+            capacity,
+        ))),
+        DataType::Decimal128(precision, scale) => Ok(XmlFieldBuilder::Decimal128 {
+            builder: Decimal128Builder::with_capacity(capacity)
+                .with_precision_and_scale(*precision, *scale)
+                .map_err(|e| DataFusionError::Internal(format!("Decimal128 builder error: {e}")))?,
+            precision: *precision,
+            scale: *scale,
+        }),
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(XmlFieldBuilder::String(
+            StringBuilder::with_capacity(capacity, capacity * 16),
+        )),
+        DataType::Date32 => Ok(XmlFieldBuilder::Date32(Date32Builder::with_capacity(
+            capacity,
+        ))),
+        DataType::Timestamp(unit, tz) => match unit {
+            TimeUnit::Microsecond => Ok(XmlFieldBuilder::TimestampMicrosecond {
+                builder: TimestampMicrosecondBuilder::with_capacity(capacity)
+                    .with_timezone_opt(tz.clone()),
+                has_tz: tz.is_some(),
+            }),
+            TimeUnit::Nanosecond => Ok(XmlFieldBuilder::TimestampNanosecond {
+                builder: TimestampNanosecondBuilder::with_capacity(capacity)
+                    .with_timezone_opt(tz.clone()),
+                has_tz: tz.is_some(),
+            }),
+            other => Ok(XmlFieldBuilder::Unsupported {
+                data_type: DataType::Timestamp(*other, tz.clone()),
+                count: 0,
+            }),
+        },
+        DataType::List(item_field) => {
+            let values = create_field_builder(item_field.data_type(), capacity, session_timezone)?;
+            let mut offsets = Vec::with_capacity(capacity + 1);
+            offsets.push(0_i32);
+            Ok(XmlFieldBuilder::List {
+                field: item_field.clone(),
+                offsets,
+                values: Box::new(values),
+                nulls: Vec::with_capacity(capacity),
+            })
+        }
+        DataType::Struct(fields) => {
+            let children = fields
+                .iter()
+                .map(|f| create_field_builder(f.data_type(), capacity, session_timezone))
+                .collect::>>()?;
+            Ok(XmlFieldBuilder::Struct {
+                fields: fields.clone(),
+                children,
+                nulls: Vec::with_capacity(capacity),
+            })
+        }
+        other => Ok(XmlFieldBuilder::Unsupported {
+            data_type: other.clone(),
+            count: 0,
+        }),
+    }
+}
+
+fn create_struct_builder(
+    fields: &Fields,
+    capacity: usize,
+    session_timezone: &str,
+) -> Result {
+    let children = fields
+        .iter()
+        .map(|f| create_field_builder(f.data_type(), capacity, session_timezone))
+        .collect::>>()?;
+    Ok(TopLevelStructBuilder {
+        fields: fields.clone(),
+        children,
+        nulls: Vec::with_capacity(capacity),
+    })
+}
+
+fn append_null_struct(builder: &mut TopLevelStructBuilder) -> Result<()> {
+    builder.nulls.push(false);
+    for child in builder.children.iter_mut() {
+        append_null_to_field(child)?;
+    }
+    Ok(())
+}
+
+fn append_null_children_struct(builder: &mut TopLevelStructBuilder) -> Result<()> {
+    builder.nulls.push(true);
+    for child in builder.children.iter_mut() {
+        append_null_to_field(child)?;
+    }
+    Ok(())
+}
+
+fn append_null_to_field(builder: &mut XmlFieldBuilder) -> Result<()> {
+    match builder {
+        XmlFieldBuilder::Boolean(b) => b.append_null(),
+        XmlFieldBuilder::Int8(b) => b.append_null(),
+        XmlFieldBuilder::Int16(b) => b.append_null(),
+        XmlFieldBuilder::Int32(b) => b.append_null(),
+        XmlFieldBuilder::Int64(b) => b.append_null(),
+        XmlFieldBuilder::Float32(b) => b.append_null(),
+        XmlFieldBuilder::Float64(b) => b.append_null(),
+        XmlFieldBuilder::Decimal128 { builder: b, .. } => b.append_null(),
+        XmlFieldBuilder::String(b) => b.append_null(),
+        XmlFieldBuilder::Date32(b) => b.append_null(),
+        XmlFieldBuilder::TimestampMicrosecond { builder: b, .. } => b.append_null(),
+        XmlFieldBuilder::TimestampNanosecond { builder: b, .. } => b.append_null(),
+        XmlFieldBuilder::List { offsets, nulls, .. } => {
+            nulls.push(false);
+            let curr = offsets.last().copied().unwrap_or(0);
+            offsets.push(curr);
+        }
+        XmlFieldBuilder::Struct {
+            children, nulls, ..
+        } => {
+            nulls.push(false);
+            for child in children.iter_mut() {
+                append_null_to_field(child)?;
+            }
+        }
+        XmlFieldBuilder::Unsupported { count, .. } => *count += 1,
+    }
+    Ok(())
+}
+
+fn parse_xml_into_builder(
+    xml: &str,
+    builder: &mut TopLevelStructBuilder,
+    options: &SparkFromXmlOptions,
+    session_timezone: &str,
+) -> Result<()> {
+    let mut documents = Documents::new();
+    let handle = documents
+        .add_string_without_uri(xml)
+        .map_err(|e| DataFusionError::Execution(format!("Invalid XML: {e}")))?;
+
+    let doc_node = documents
+        .document_node(handle)
+        .ok_or_else(|| DataFusionError::Execution("XML document has no root node".to_string()))?;
+
+    let xot = documents.xot();
+    let root = xot
+        .document_element(doc_node)
+        .map_err(|e| DataFusionError::Execution(format!("No root element: {e}")))?;
+
+    builder.nulls.push(true);
+
+    for (field, child_builder) in builder.fields.iter().zip(builder.children.iter_mut()) {
+        append_xml_field(
+            xot,
+            root,
+            field.name(),
+            child_builder,
+            options,
+            session_timezone,
+        )?;
+    }
+
+    Ok(())
+}
+
+fn append_xml_field(
+    xot: &xot::Xot,
+    parent: xot::Node,
+    field_name: &str,
+    builder: &mut XmlFieldBuilder,
+    options: &SparkFromXmlOptions,
+    session_timezone: &str,
+) -> Result<()> {
+    if field_name == options.value_tag {
+        let text = element_text_content(xot, parent);
+        return append_text(text.as_deref(), builder, options, session_timezone);
+    }
+
+    let attr_name = field_name.strip_prefix(options.attribute_prefix.as_str());
+    if let Some(attr_name) = attr_name {
+        if let Some(val) = find_attribute(xot, parent, attr_name) {
+            return append_text(Some(val.as_str()), builder, options, session_timezone);
+        } else if !options.attribute_prefix.is_empty() {
+            return append_text(None, builder, options, session_timezone);
+        }
+    }
+
+    let matches: Vec = child_elements_named(xot, parent, field_name);
+
+    match builder {
+        XmlFieldBuilder::List {
+            field,
+            offsets,
+            values,
+            nulls,
+        } => {
+            if matches.is_empty() {
+                nulls.push(false);
+                let curr = offsets.last().copied().unwrap_or(0);
+                offsets.push(curr);
+            } else {
+                nulls.push(true);
+                let start = offsets.last().copied().unwrap_or(0);
+                for &node in &matches {
+                    append_element_to_field(xot, node, values, options, session_timezone)?;
+                }
+                offsets.push(start + matches.len() as i32);
+                let _ = field;
+            }
+        }
+        XmlFieldBuilder::Struct {
+            fields,
+            children,
+            nulls,
+        } => {
+            if matches.is_empty() {
+                nulls.push(false);
+                for child in children.iter_mut() {
+                    append_null_to_field(child)?;
+                }
+            } else {
+                nulls.push(true);
+                let node = matches[0];
+                for (f, child) in fields.iter().zip(children.iter_mut()) {
+                    append_xml_field(xot, node, f.name(), child, options, session_timezone)?;
+                }
+            }
+        }
+        _ => {
+            if matches.is_empty() {
+                append_null_to_field(builder)?;
+            } else {
+                let text = element_text_content(xot, matches[0]);
+                append_text(text.as_deref(), builder, options, session_timezone)?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn append_element_to_field(
+    xot: &xot::Xot,
+    node: xot::Node,
+    builder: &mut XmlFieldBuilder,
+    options: &SparkFromXmlOptions,
+    session_timezone: &str,
+) -> Result<()> {
+    match builder {
+        XmlFieldBuilder::Struct {
+            fields,
+            children,
+            nulls,
+        } => {
+            nulls.push(true);
+            for (f, child) in fields.iter().zip(children.iter_mut()) {
+                append_xml_field(xot, node, f.name(), child, options, session_timezone)?;
+            }
+        }
+        _ => {
+            let text = element_text_content(xot, node);
+            append_text(text.as_deref(), builder, options, session_timezone)?;
+        }
+    }
+    Ok(())
+}
+
+fn append_text(
+    text: Option<&str>,
+    builder: &mut XmlFieldBuilder,
+    options: &SparkFromXmlOptions,
+    session_timezone: &str,
+) -> Result<()> {
+    let raw = match text {
+        None => return append_null_to_field(builder),
+        Some(s) => s,
+    };
+
+    if let Some(nv) = &options.null_value
+        && raw == nv.as_str()
+    {
+        return append_null_to_field(builder);
+    }
+
+    match builder {
+        XmlFieldBuilder::Boolean(b) => match raw.to_ascii_lowercase().as_str() {
+            "true" | "1" => b.append_value(true),
+            "false" | "0" => b.append_value(false),
+            _ => b.append_null(),
+        },
+        XmlFieldBuilder::Int8(b) => match raw.parse::() {
+            Ok(v) => b.append_value(v),
+            Err(_) => b.append_null(),
+        },
+        XmlFieldBuilder::Int16(b) => match raw.parse::() {
+            Ok(v) => b.append_value(v),
+            Err(_) => b.append_null(),
+        },
+        XmlFieldBuilder::Int32(b) => match raw.parse::() {
+            Ok(v) => b.append_value(v),
+            Err(_) => b.append_null(),
+        },
+        XmlFieldBuilder::Int64(b) => match raw.parse::() {
+            Ok(v) => b.append_value(v),
+            Err(_) => b.append_null(),
+        },
+        XmlFieldBuilder::Float32(b) => match raw.parse::() {
+            Ok(v) => b.append_value(v),
+            Err(_) => b.append_null(),
+        },
+        XmlFieldBuilder::Decimal128 {
+            builder: b,
+            precision,
+            scale,
+        } => match parse_decimal128(raw, *precision, *scale) {
+            Some(v) => b.append_value(v),
+            None => b.append_null(),
+        },
+        XmlFieldBuilder::Float64(b) => match raw.parse::() {
+            Ok(v) => b.append_value(v),
+            Err(_) => b.append_null(),
+        },
+
+        XmlFieldBuilder::String(b) => b.append_value(raw),
+        XmlFieldBuilder::Date32(b) => match options.date_format.parse_date_value(raw) {
+            Ok(d) => b.append_value(Date32Type::from_naive_date(d)),
+            Err(_) => b.append_null(),
+        },
+        XmlFieldBuilder::TimestampMicrosecond { builder: b, has_tz } => {
+            let fmt = if *has_tz {
+                &options.timestamp_ltz_format
+            } else {
+                &options.timestamp_ntz_format
+            };
+            match fmt.parse_datetime_value(raw) {
+                Ok(parsed) => {
+                    let micros = parsed_timestamp_to_micros(parsed, *has_tz, session_timezone)?;
+                    b.append_value(micros);
+                }
+                Err(_) => b.append_null(),
+            }
+        }
+        XmlFieldBuilder::TimestampNanosecond { builder: b, has_tz } => {
+            let fmt = if *has_tz {
+                &options.timestamp_ltz_format
+            } else {
+                &options.timestamp_ntz_format
+            };
+            match fmt.parse_datetime_value(raw) {
+                Ok(parsed) => {
+                    let micros = parsed_timestamp_to_micros(parsed, *has_tz, session_timezone)?;
+                    b.append_value(micros * 1_000);
+                }
+                Err(_) => b.append_null(),
+            }
+        }
+        XmlFieldBuilder::List { .. } | XmlFieldBuilder::Struct { .. } => {
+            append_null_to_field(builder)?;
+        }
+        XmlFieldBuilder::Unsupported { count, .. } => *count += 1,
+    }
+    Ok(())
+}
+
+fn parsed_timestamp_to_micros(
+    parsed: ParsedDateTime,
+    has_timezone: bool,
+    session_timezone: &str,
+) -> Result {
+    if !has_timezone {
+        return Ok(parsed.datetime.and_utc().timestamp_micros());
+    }
+
+    if let Some(offset) = parsed.offset {
+        return parsed
+            .datetime
+            .and_local_timezone(offset)
+            .single()
+            .map(|datetime| datetime.to_utc().timestamp_micros())
+            .ok_or_else(|| DataFusionError::Execution("cannot apply parsed offset".to_string()));
+    }
+
+    let timezone_name = parsed.timezone.as_deref().unwrap_or(session_timezone);
+    let timezone: Tz = timezone_name.parse().map_err(|e| {
+        DataFusionError::Execution(format!("Invalid timezone '{timezone_name}': {e}"))
+    })?;
+    Ok(localize_with_fallback(&timezone, &parsed.datetime)?.timestamp_micros())
+}
+
+fn parse_decimal128(raw: &str, precision: u8, scale: i8) -> Option {
+    let raw = raw.trim();
+    let is_negative = raw.starts_with('-');
+    let raw = if is_negative { &raw[1..] } else { raw };
+
+    let (int_part, frac_part) = if let Some(pos) = raw.find('.') {
+        (&raw[..pos], &raw[pos + 1..])
+    } else {
+        (raw, "")
+    };
+
+    let int_val: i128 = int_part.parse().ok()?;
+
+    let scale = scale as usize;
+    let frac_val: i128 = if frac_part.len() >= scale {
+        frac_part[..scale].parse().ok()?
+    } else {
+        let padded = format!("{:0= max_value {
+        return None; // overflow
+    }
+    Some(value)
+}
+
+fn child_elements_named(xot: &xot::Xot, parent: xot::Node, name: &str) -> Vec {
+    xot.children(parent)
+        .filter(|&child| {
+            if let xot::Value::Element(el) = xot.value(child) {
+                xot.local_name_str(el.name()) == name
+            } else {
+                false
+            }
+        })
+        .collect()
+}
+
+fn element_text_content(xot: &xot::Xot, node: xot::Node) -> Option {
+    let text: String = xot
+        .children(node)
+        .filter_map(|child| {
+            if let xot::Value::Text(t) = xot.value(child) {
+                Some(t.get().to_string())
+            } else {
+                None
+            }
+        })
+        .collect();
+    Some(text)
+}
+
+fn find_attribute(xot: &xot::Xot, node: xot::Node, attr_name: &str) -> Option {
+    for (name_id, value) in xot.attributes(node).iter() {
+        if xot.local_name_str(name_id) == attr_name {
+            return Some(value.clone());
+        }
+    }
+    None
+}
+
+fn find_key_value(map: &MapArray, key: &str) -> Option {
+    if map.is_empty() {
+        return None;
+    }
+    let entries = map.value(0);
+    let keys = entries
+        .column_by_name(SAIL_MAP_KEY_FIELD_NAME)?
+        .as_any()
+        .downcast_ref::()?;
+    let values = entries
+        .column_by_name(SAIL_MAP_VALUE_FIELD_NAME)?
+        .as_any()
+        .downcast_ref::()?;
+    for i in 0..keys.len() {
+        if !keys.is_null(i) && keys.value(i).eq_ignore_ascii_case(key) {
+            return if values.is_null(i) {
+                None
+            } else {
+                Some(values.value(i).to_string())
+            };
+        }
+    }
+    None
+}
+
+fn finish_field_builder(builder: XmlFieldBuilder) -> Result {
+    match builder {
+        XmlFieldBuilder::Boolean(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Int8(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Int16(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Int32(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Int64(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Float32(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Float64(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Decimal128 { mut builder, .. } => Ok(Arc::new(builder.finish())),
+        XmlFieldBuilder::String(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::Date32(mut b) => Ok(Arc::new(b.finish())),
+        XmlFieldBuilder::TimestampMicrosecond { mut builder, .. } => Ok(Arc::new(builder.finish())),
+        XmlFieldBuilder::TimestampNanosecond { mut builder, .. } => Ok(Arc::new(builder.finish())),
+        XmlFieldBuilder::List {
+            field,
+            offsets,
+            values,
+            nulls,
+        } => {
+            let values_array = finish_field_builder(*values)?;
+            Ok(Arc::new(ListArray::new(
+                field,
+                OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                values_array,
+                Some(NullBuffer::from(nulls)),
+            )))
+        }
+        XmlFieldBuilder::Struct {
+            fields,
+            children,
+            nulls,
+        } => {
+            let arrays = children
+                .into_iter()
+                .map(finish_field_builder)
+                .collect::>>()?;
+            Ok(Arc::new(StructArray::new(
+                fields,
+                arrays,
+                Some(NullBuffer::from(nulls)),
+            )))
+        }
+        XmlFieldBuilder::Unsupported { data_type, count } => Ok(new_null_array(&data_type, count)),
+    }
+}
+
+fn finish_struct_builder(builder: TopLevelStructBuilder) -> Result {
+    let arrays = builder
+        .children
+        .into_iter()
+        .map(finish_field_builder)
+        .collect::>>()?;
+    Ok(Arc::new(StructArray::new(
+        builder.fields,
+        arrays,
+        Some(NullBuffer::from(builder.nulls)),
+    )))
+}
+
+#[cfg(test)]
+#[expect(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    fn run(xml: &str, schema: &str) -> Result {
+        let xml_arr = Arc::new(StringArray::from(vec![Some(xml)])) as ArrayRef;
+        let schema_arr = Arc::new(StringArray::from(vec![schema])) as ArrayRef;
+        spark_from_xml_inner(&[xml_arr, schema_arr], DEFAULT_SESSION_TIMEZONE)
+    }
+
+    fn col(result: &ArrayRef, name: &str) -> ArrayRef {
+        result
+            .as_any()
+            .downcast_ref::()
+            .unwrap()
+            .column_by_name(name)
+            .unwrap()
+            .clone()
+    }
+
+    #[test]
+    fn test_primitives() -> Result<()> {
+        let r = run("
10.8
", "a INT, b DOUBLE")?;
+        assert_eq!(
+            col(&r, "a")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            1
+        );
+        assert!(
+            (col(&r, "b")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0)
+                - 0.8)
+                .abs()
+                < 1e-9
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_missing_tag_is_null() -> Result<()> {
+        let r = run("
1
", "a INT, b INT")?;
+        assert!(col(&r, "a").is_null(0));
+        assert_eq!(
+            col(&r, "b")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            1
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_empty_tag_string_is_empty_string() -> Result<()> {
+        let r = run("
", "a STRING")?;
+        assert_eq!(
+            col(&r, "a")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            ""
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_empty_tag_int_is_null() -> Result<()> {
+        let r = run("
", "a INT")?;
+        assert!(col(&r, "a").is_null(0));
+        Ok(())
+    }
+
+    #[test]
+    fn test_null_input_is_null_struct() -> Result<()> {
+        let xml_arr = Arc::new(StringArray::from(vec![None::<&str>])) as ArrayRef;
+        let schema_arr = Arc::new(StringArray::from(vec!["a INT"])) as ArrayRef;
+        let r = spark_from_xml_inner(&[xml_arr, schema_arr], DEFAULT_SESSION_TIMEZONE)?;
+        assert!(r.is_null(0));
+        Ok(())
+    }
+
+    #[test]
+    fn test_root_tag_ignored() -> Result<()> {
+        for xml in &[
+            "1",
+            "1",
+            "
1
",
+        ] {
+            let r = run(xml, "a INT")?;
+            assert_eq!(
+                col(&r, "a")
+                    .as_any()
+                    .downcast_ref::()
+                    .unwrap()
+                    .value(0),
+                1
+            );
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_xml_entities_unescaped() -> Result<()> {
+        let r = run("
a < b
", "s STRING")?;
+        assert_eq!(
+            col(&r, "s")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            "a < b"
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_attribute_prefix() -> Result<()> {
+        let r = run(
+            r#"
Alice
"#,
+            "_id STRING, name STRING",
+        )?;
+        assert_eq!(
+            col(&r, "_id")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            "99"
+        );
+        assert_eq!(
+            col(&r, "name")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            "Alice"
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_repeated_tags_array() -> Result<()> {
+        let r = run("
123
", "a ARRAY")?;
+        let list = col(&r, "a")
+            .as_any()
+            .downcast_ref::()
+            .unwrap()
+            .value(0);
+        let ints = list.as_any().downcast_ref::().unwrap();
+        assert_eq!(ints.len(), 3);
+        assert_eq!(ints.value(0), 1);
+        assert_eq!(ints.value(1), 2);
+        assert_eq!(ints.value(2), 3);
+        Ok(())
+    }
+
+    #[test]
+    fn test_nested_struct() -> Result<()> {
+        let r = run(
+            "
Bob1
",
+            "student STRUCT",
+        )?;
+        let nested = col(&r, "student")
+            .as_any()
+            .downcast_ref::()
+            .unwrap()
+            .clone();
+        let name = nested.column_by_name("name").unwrap();
+        assert_eq!(
+            name.as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0),
+            "Bob"
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_boolean() -> Result<()> {
+        let r = run("
true
", "flag BOOLEAN")?;
+        assert!(
+            col(&r, "flag")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0)
+        );
+        let r = run("
false
", "flag BOOLEAN")?;
+        assert!(
+            !col(&r, "flag")
+                .as_any()
+                .downcast_ref::()
+                .unwrap()
+                .value(0)
+        );
+        Ok(())
+    }
+}
diff --git a/crates/sail-function/src/scalar/xml/mod.rs b/crates/sail-function/src/scalar/xml/mod.rs
index 032c9a1ae..ad4fe1439 100644
--- a/crates/sail-function/src/scalar/xml/mod.rs
+++ b/crates/sail-function/src/scalar/xml/mod.rs
@@ -1,3 +1,4 @@
+pub mod from_xml;
 pub mod to_xml;
 pub mod xpath;
 pub mod xpath_typed;
diff --git a/crates/sail-function/src/scalar/xml/to_xml.rs b/crates/sail-function/src/scalar/xml/to_xml.rs
index f0be4577c..69d949f4f 100644
--- a/crates/sail-function/src/scalar/xml/to_xml.rs
+++ b/crates/sail-function/src/scalar/xml/to_xml.rs
@@ -5,12 +5,12 @@ use chrono::prelude::*;
 use datafusion::arrow::array::timezone::Tz;
 use datafusion::arrow::array::*;
 use datafusion::arrow::datatypes::*;
-use datafusion_common::{exec_err, plan_err, DataFusionError, Result, ScalarValue};
+use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature};
 use datafusion_expr_common::signature::Volatility;
 use sail_common::spec::{SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME};
 
-use crate::scalar::datetime::utils::spark_datetime_format_to_chrono_strftime;
+use crate::scalar::datetime::format::DateTimeFormat;
 
 /// Spark-compatible `to_xml` UDF. Serializes a StructArray into XML strings.
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -27,10 +27,9 @@ struct SparkToXmlOptions {
     value_tag: String,
     /// If Some(s), nulls are emitted as s. If None, they are omitted.
     null_value: Option,
-    declaration: String,
-    timestamp_ltz_format: Option,
-    timestamp_ntz_format: String,
-    date_format: String,
+    timestamp_ltz_format: Option,
+    timestamp_ntz_format: DateTimeFormat,
+    date_format: DateTimeFormat,
     session_timezone: String,
 }
 
@@ -44,17 +43,14 @@ impl SparkToXmlOptions {
     pub const VALUE_TAG_OPTION: &'static str = "valueTag";
     pub const VALUE_TAG_DEFAULT: &'static str = "_VALUE";
     pub const NULL_VALUE_OPTION: &'static str = "nullValue";
-    pub const DECLARATION_OPTION: &'static str = "declaration";
-    pub const DECLARATION_DEFAULT: &'static str =
-        r#"version="1.0" encoding="UTF-8" standalone="yes""#;
     pub const TIMESTAMP_FORMAT_OPTION: &'static str = "timestampFormat";
     pub const TIMESTAMP_NTZ_FORMAT_OPTION: &'static str = "timestampNTZFormat";
     pub const DATE_FORMAT_OPTION: &'static str = "dateFormat";
 
-    // Standard Spark strftime defaults
-    pub const TIMESTAMP_LTZ_FORMAT_DEFAULT: &'static str = "%Y-%m-%dT%H:%M:%S%.3f%:z";
-    pub const TIMESTAMP_NTZ_FORMAT_DEFAULT: &'static str = "%Y-%m-%dT%H:%M:%S%.3f";
-    pub const DATE_FORMAT_DEFAULT: &'static str = "%Y-%m-%d";
+    // Standard Spark Java DateTimeFormatter defaults
+    pub const TIMESTAMP_LTZ_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
+    pub const TIMESTAMP_NTZ_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+    pub const DATE_FORMAT_DEFAULT: &'static str = "yyyy-MM-dd";
 
     fn from_map(map: &MapArray, session_timezone: &str) -> Result {
         let row_tag = find_key_value(map, Self::ROW_TAG_OPTION)
@@ -66,12 +62,6 @@ impl SparkToXmlOptions {
             return plan_err!("`rowTag` must not include angle brackets");
         }
 
-        let declaration = find_key_value(map, Self::DECLARATION_OPTION)
-            .unwrap_or_else(|| Self::DECLARATION_DEFAULT.to_string());
-        if declaration.starts_with('<') || declaration.ends_with('>') {
-            return plan_err!("`declaration` must not include angle brackets");
-        }
-
         let attribute_prefix = find_key_value(map, Self::ATTRIBUTE_PREFIX_OPTION)
             .unwrap_or_else(|| Self::ATTRIBUTE_PREFIX_DEFAULT.to_string());
         if attribute_prefix.is_empty() {
@@ -91,20 +81,28 @@ impl SparkToXmlOptions {
 
         let timestamp_ltz_format = find_key_value(map, Self::TIMESTAMP_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?;
 
         let timestamp_ntz_format = find_key_value(map, Self::TIMESTAMP_NTZ_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
-            .unwrap_or_else(|| Self::TIMESTAMP_NTZ_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_formatting(Self::TIMESTAMP_NTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp NTZ format should be valid")
+            });
 
         let date_format = find_key_value(map, Self::DATE_FORMAT_OPTION)
             .as_deref()
-            .map(spark_datetime_format_to_chrono_strftime)
+            .map(DateTimeFormat::for_formatting)
             .transpose()?
-            .unwrap_or_else(|| Self::DATE_FORMAT_DEFAULT.to_string());
+            .unwrap_or_else(|| {
+                #[expect(clippy::expect_used)]
+                DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
+                    .expect("default date format should be valid")
+            });
 
         Ok(Self {
             row_tag,
@@ -113,7 +111,6 @@ impl SparkToXmlOptions {
             attribute_prefix,
             value_tag,
             null_value,
-            declaration,
             timestamp_ltz_format,
             timestamp_ntz_format,
             date_format,
@@ -134,6 +131,7 @@ impl SparkToXmlOptions {
 }
 
 impl Default for SparkToXmlOptions {
+    #[expect(clippy::expect_used)]
     fn default() -> Self {
         Self {
             row_tag: Self::ROW_TAG_DEFAULT.to_string(),
@@ -141,10 +139,13 @@ impl Default for SparkToXmlOptions {
             attribute_prefix: Self::ATTRIBUTE_PREFIX_DEFAULT.to_string(),
             value_tag: Self::VALUE_TAG_DEFAULT.to_string(),
             null_value: None,
-            declaration: Self::DECLARATION_DEFAULT.to_string(),
             timestamp_ltz_format: None,
-            timestamp_ntz_format: Self::TIMESTAMP_NTZ_FORMAT_DEFAULT.to_string(),
-            date_format: Self::DATE_FORMAT_DEFAULT.to_string(),
+            timestamp_ntz_format: DateTimeFormat::for_formatting(
+                Self::TIMESTAMP_NTZ_FORMAT_DEFAULT,
+            )
+            .expect("default timestamp NTZ format should be valid"),
+            date_format: DateTimeFormat::for_formatting(Self::DATE_FORMAT_DEFAULT)
+                .expect("default date format should be valid"),
             session_timezone: "UTC".to_string(),
         }
     }
@@ -205,7 +206,7 @@ impl ScalarUDFImpl for SparkToXml {
                 match &arg_types[0] {
                     DataType::Struct(_) => {}
                     other => {
-                        return plan_err!("`to_xml` first argument must be a struct, got {other}")
+                        return plan_err!("`to_xml` first argument must be a struct, got {other}");
                     }
                 }
                 match &arg_types[1] {
@@ -213,7 +214,7 @@ impl ScalarUDFImpl for SparkToXml {
                     other => {
                         return plan_err!(
                             "`to_xml` second argument must be a map of options, got {other}"
-                        )
+                        );
                     }
                 }
                 Ok(arg_types.to_vec())
@@ -257,39 +258,50 @@ impl ScalarUDFImpl for SparkToXml {
 }
 
 fn spark_to_xml_inner(array: &StructArray, options: &SparkToXmlOptions) -> Result {
-    let mut output: Vec> = Vec::with_capacity(array.len());
+    let mut builder = StringBuilder::with_capacity(array.len(), 0);
+    // One reused buffer across rows: `clear()` keeps its capacity, so the row
+    // strings are not re-allocated per row and are copied into the Arrow buffer
+    // exactly once.
+    let mut buf = String::with_capacity(256);
 
     for row in 0..array.len() {
         if array.is_null(row) {
-            output.push(None);
+            builder.append_null();
         } else {
-            output.push(Some(write_row(array, row, array.fields(), options)?));
+            buf.clear();
+            write_struct(
+                &mut buf,
+                array,
+                row,
+                array.fields(),
+                &options.row_tag,
+                0,
+                options,
+            )?;
+            // Every writer prefixes its own leading newline; the root element has none.
+            builder.append_value(buf.strip_prefix('\n').unwrap_or(&buf));
         }
     }
 
-    Ok(Arc::new(StringArray::from(output)))
+    Ok(Arc::new(builder.finish()))
 }
 
 const INDENT: &str = "    ";
 
-fn write_row(
-    array: &StructArray,
-    row: usize,
-    fields: &Fields,
-    options: &SparkToXmlOptions,
-) -> Result {
-    let mut buf = String::with_capacity(256);
-    if !options.declaration.is_empty() {
-        buf.push_str("\n");
+/// Appends `depth` levels of indentation without allocating (unlike
+/// `INDENT.repeat(depth)`, which builds a fresh `String` each call).
+#[inline]
+fn push_indent(buf: &mut String, depth: usize) {
+    for _ in 0..depth {
+        buf.push_str(INDENT);
     }
-    write_struct(&mut buf, array, row, fields, &options.row_tag, 0, options)?;
-    Ok(buf)
 }
 
-/// Matches Spark's two-pass logic: attributes are written to the open tag,
-/// then child elements follow.
+/// Serializes a struct as `…children…`, matching Spark's
+/// `StaxXmlGenerator`. Uses a leading-newline convention: every element/array/map
+/// child prefixes its own `\n` plus indentation, so no trailing newline is ever
+/// appended. Attributes go on the open tag; the `valueTag` field becomes inline
+/// text right after `>`; remaining fields become child elements.
 fn write_struct(
     buf: &mut String,
     array: &StructArray,
@@ -299,19 +311,17 @@ fn write_struct(
     depth: usize,
     options: &SparkToXmlOptions,
 ) -> Result<()> {
-    let pad = INDENT.repeat(depth);
-
-    let (attr_cols, elem_cols): (Vec<_>, Vec<_>) = fields
-        .iter()
-        .enumerate()
-        .partition(|(_, f)| options.is_attribute(f.name()));
-
-    buf.push_str(&pad);
+    buf.push('\n');
+    push_indent(buf, depth);
     buf.push('<');
     buf.push_str(tag);
 
-    for (col_idx, field) in &attr_cols {
-        let col = array.column(*col_idx);
+    // First pass: attributes go on the open tag.
+    for (col_idx, field) in fields.iter().enumerate() {
+        if !options.is_attribute(field.name()) {
+            continue;
+        }
+        let col = array.column(col_idx);
         if col.is_null(row) {
             if let Some(ref nv) = options.null_value {
                 let attr_name = options.strip_prefix(field.name());
@@ -324,27 +334,51 @@ fn write_struct(
         }
     }
 
-    if elem_cols.is_empty() {
-        buf.push_str("/>\n");
-    } else {
-        let content_start = buf.len();
-        buf.push_str(">\n");
-        for (col_idx, field) in &elem_cols {
-            let col = array.column(*col_idx);
-            write_field(buf, col, row, field, depth + 1, options)?;
-        }
-        let content_end = buf.len();
-        if content_end == content_start + 2 {
-            buf.truncate(content_start);
-            buf.push_str("/>\n");
+    // Second pass: the valueTag field becomes inline text, the rest child elements.
+    let mut value_text: Option = None;
+    let mut children = String::new();
+    for (col_idx, field) in fields.iter().enumerate() {
+        if options.is_attribute(field.name()) {
+            continue;
+        }
+        let col = array.column(col_idx);
+        if field.name() == options.value_tag.as_str() {
+            if col.is_null(row) {
+                if let Some(ref nv) = options.null_value {
+                    value_text = Some(escape_text(nv));
+                }
+            } else {
+                value_text = Some(escape_text(&format_field_to_xml(
+                    col,
+                    row,
+                    field.data_type(),
+                    options,
+                )?));
+            }
         } else {
-            buf.push_str(&pad);
-            buf.push_str("\n");
+            write_field(&mut children, col, row, field, depth + 1, options)?;
         }
     }
 
+    let has_text = value_text.is_some();
+    let has_children = !children.is_empty();
+    if !has_text && !has_children {
+        buf.push_str("/>");
+    } else {
+        buf.push('>');
+        if let Some(text) = value_text {
+            buf.push_str(&text);
+        }
+        buf.push_str(&children);
+        if has_children {
+            buf.push('\n');
+            push_indent(buf, depth);
+        }
+        buf.push_str("');
+    }
+
     Ok(())
 }
 
@@ -356,31 +390,13 @@ fn write_field(
     depth: usize,
     options: &SparkToXmlOptions,
 ) -> Result<()> {
-    let pad = INDENT.repeat(depth);
     let name = field.name().as_str();
 
-    // value_tag handles text content injected directly into the parent
-    if name == options.value_tag {
-        if col.is_null(row) {
-            if let Some(ref nv) = options.null_value {
-                buf.push_str(&escape_text(nv));
-            }
-        } else {
-            buf.push_str(&escape_text(&format_field_to_xml(
-                col,
-                row,
-                field.data_type(),
-                options,
-            )?));
-        }
-        return Ok(());
-    }
-
     if col.is_null(row) {
         if let Some(ref nv) = options.null_value {
-            buf.push_str(&pad);
-            push_element(buf, name, &escape_text(nv));
             buf.push('\n');
+            push_indent(buf, depth);
+            push_element(buf, name, &escape_text(nv));
         }
         return Ok(());
     }
@@ -402,15 +418,18 @@ fn write_field(
         }
         _ => {
             let text = format_field_to_xml(col, row, field.data_type(), options)?;
-            buf.push_str(&pad);
-            push_element(buf, name, &escape_text(&text));
             buf.push('\n');
+            push_indent(buf, depth);
+            push_element(buf, name, &escape_text(&text));
         }
     }
 
     Ok(())
 }
 
+/// Serializes an array field. Spark repeats the field tag once per element for
+/// primitives (`12`) and structs, with no `item` wrapper. Only a
+/// nested array wraps its inner elements, which then use `arrayElementName`.
 fn write_array(
     buf: &mut String,
     col: &ArrayRef,
@@ -420,8 +439,6 @@ fn write_array(
     depth: usize,
     options: &SparkToXmlOptions,
 ) -> Result<()> {
-    let pad = INDENT.repeat(depth);
-
     let (offsets_start, offsets_end, values): (usize, usize, ArrayRef) =
         if let Some(list) = col.as_any().downcast_ref::() {
             (
@@ -461,45 +478,39 @@ fn write_array(
                 )?;
             }
             DataType::List(inner_field) | DataType::LargeList(inner_field) => {
+                buf.push('\n');
+                push_indent(buf, depth);
+                buf.push('<');
+                buf.push_str(field_name);
+                buf.push('>');
                 write_array(
                     buf,
                     &values,
                     item_idx,
                     options.array_element_name.as_str(),
                     inner_field,
-                    depth,
+                    depth + 1,
                     options,
                 )?;
+                buf.push('\n');
+                push_indent(buf, depth);
+                buf.push_str("');
             }
             _ => {
                 if values.is_null(item_idx) {
                     if let Some(ref nv) = options.null_value {
-                        buf.push_str(&pad);
-                        buf.push('<');
-                        buf.push_str(field_name);
-                        buf.push_str(">\n");
-                        buf.push_str(&INDENT.repeat(depth + 1));
-                        push_element(buf, &options.array_element_name, &escape_text(nv));
                         buf.push('\n');
-                        buf.push_str(&pad);
-                        buf.push_str("\n");
+                        push_indent(buf, depth);
+                        push_element(buf, field_name, &escape_text(nv));
                     }
                 } else {
                     let text =
                         format_field_to_xml(&values, item_idx, item_field.data_type(), options)?;
-                    buf.push_str(&pad);
-                    buf.push('<');
-                    buf.push_str(field_name);
-                    buf.push_str(">\n");
-                    buf.push_str(&INDENT.repeat(depth + 1));
-                    push_element(buf, &options.array_element_name, &escape_text(&text));
                     buf.push('\n');
-                    buf.push_str(&pad);
-                    buf.push_str("\n");
+                    push_indent(buf, depth);
+                    push_element(buf, field_name, &escape_text(&text));
                 }
             }
         }
@@ -522,8 +533,6 @@ fn write_map(
         ))
     })?;
 
-    let pad = INDENT.repeat(depth);
-
     let entries = map_array.value(row);
     let num_entries = entries.len();
     let keys = entries
@@ -557,7 +566,7 @@ fn write_map(
             other => {
                 return exec_err!(
                     "to_xml: map keys must be strings to be valid XML tag names, got {other:?}"
-                )
+                );
             }
         };
         if key.is_empty() {
@@ -573,112 +582,113 @@ fn write_map(
                  which is not a valid XML name start character"
             );
         }
-        if values.is_null(i) {
-            if let Some(ref nv) = options.null_value {
-                inner.push_str(&INDENT.repeat(depth + 1));
-                push_element(&mut inner, &key, &escape_text(nv));
-                inner.push('\n');
-            }
-        } else {
-            let val_dt = values.data_type();
-            match val_dt {
-                DataType::Struct(child_fields) => {
-                    let child = values
+        write_map_entry(&mut inner, values, i, &key, depth + 1, options)?;
+    }
+
+    buf.push('\n');
+    push_indent(buf, depth);
+    buf.push('<');
+    buf.push_str(field_name);
+    if inner.is_empty() {
+        buf.push_str("/>");
+    } else {
+        buf.push('>');
+        buf.push_str(&inner);
+        buf.push('\n');
+        push_indent(buf, depth);
+        buf.push_str("');
+    }
+
+    Ok(())
+}
+
+/// Renders a single map value under tag `key`, prefixing its own newline plus
+/// indentation. A list value repeats `key` once per element (like a top-level
+/// primitive array).
+fn write_map_entry(
+    buf: &mut String,
+    values: &ArrayRef,
+    i: usize,
+    key: &str,
+    depth: usize,
+    options: &SparkToXmlOptions,
+) -> Result<()> {
+    if values.is_null(i) {
+        if let Some(ref nv) = options.null_value {
+            buf.push('\n');
+            push_indent(buf, depth);
+            push_element(buf, key, &escape_text(nv));
+        }
+        return Ok(());
+    }
+
+    match values.data_type() {
+        DataType::Struct(child_fields) => {
+            let child = values
+                .as_any()
+                .downcast_ref::()
+                .ok_or_else(|| {
+                    DataFusionError::Internal(format!(
+                        "to_xml: expected StructArray for map value '{key}'"
+                    ))
+                })?;
+            write_struct(buf, child, i, child_fields, key, depth, options)?;
+        }
+        DataType::List(item_field) | DataType::LargeList(item_field) => {
+            let (offsets_start, offsets_end, array_values): (usize, usize, ArrayRef) =
+                if let Some(list) = values.as_any().downcast_ref::() {
+                    (
+                        list.offsets()[i] as usize,
+                        list.offsets()[i + 1] as usize,
+                        list.values().clone(),
+                    )
+                } else if let Some(list) = values.as_any().downcast_ref::() {
+                    (
+                        list.offsets()[i] as usize,
+                        list.offsets()[i + 1] as usize,
+                        list.values().clone(),
+                    )
+                } else {
+                    return exec_err!("to_xml: expected ListArray for map value '{key}'");
+                };
+            for elem_idx in offsets_start..offsets_end {
+                if array_values.is_null(elem_idx) {
+                    if let Some(ref nv) = options.null_value {
+                        buf.push('\n');
+                        push_indent(buf, depth);
+                        push_element(buf, key, &escape_text(nv));
+                    }
+                } else if let DataType::Struct(child_fields) = item_field.data_type() {
+                    let child = array_values
                         .as_any()
                         .downcast_ref::()
                         .ok_or_else(|| {
                             DataFusionError::Internal(format!(
-                                "to_xml: expected StructArray for map value in '{field_name}'"
+                                "to_xml: expected StructArray in map array value '{key}'"
                             ))
                         })?;
-                    write_struct(&mut inner, child, i, child_fields, &key, depth + 1, options)?;
-                }
-                DataType::List(item_field) | DataType::LargeList(item_field) => {
-                    let (offsets_start, offsets_end, array_values): (usize, usize, ArrayRef) =
-                        if let Some(list) = values.as_any().downcast_ref::() {
-                            (
-                                list.offsets()[i] as usize,
-                                list.offsets()[i + 1] as usize,
-                                list.values().clone(),
-                            )
-                        } else if let Some(list) = values.as_any().downcast_ref::()
-                        {
-                            (
-                                list.offsets()[i] as usize,
-                                list.offsets()[i + 1] as usize,
-                                list.values().clone(),
-                            )
-                        } else {
-                            return exec_err!(
-                                "to_xml: expected ListArray for map value in field '{field_name}'"
-                            );
-                        };
-                    for elem_idx in offsets_start..offsets_end {
-                        if array_values.is_null(elem_idx) {
-                            if let Some(ref nv) = options.null_value {
-                                inner.push_str(&INDENT.repeat(depth + 1));
-                                push_element(&mut inner, &key, &escape_text(nv));
-                                inner.push('\n');
-                            }
-                        } else {
-                            match item_field.data_type() {
-                                DataType::Struct(child_fields) => {
-                                    let child = array_values
-                                        .as_any()
-                                        .downcast_ref::()
-                                        .ok_or_else(|| DataFusionError::Internal(format!(
-                                            "to_xml: expected StructArray in map array value for '{field_name}'"
-                                        )))?;
-                                    write_struct(
-                                        &mut inner,
-                                        child,
-                                        elem_idx,
-                                        child_fields,
-                                        &key,
-                                        depth + 1,
-                                        options,
-                                    )?;
-                                }
-                                _ => {
-                                    let text = format_field_to_xml(
-                                        &array_values,
-                                        elem_idx,
-                                        item_field.data_type(),
-                                        options,
-                                    )?;
-                                    inner.push_str(&INDENT.repeat(depth + 1));
-                                    push_element(&mut inner, &key, &escape_text(&text));
-                                    inner.push('\n');
-                                }
-                            }
-                        }
-                    }
-                }
-                _ => {
-                    let text = format_field_to_xml(values, i, val_dt, options)?;
-                    inner.push_str(&INDENT.repeat(depth + 1));
-                    push_element(&mut inner, &key, &escape_text(&text));
-                    inner.push('\n');
+                    write_struct(buf, child, elem_idx, child_fields, key, depth, options)?;
+                } else {
+                    let text = format_field_to_xml(
+                        &array_values,
+                        elem_idx,
+                        item_field.data_type(),
+                        options,
+                    )?;
+                    buf.push('\n');
+                    push_indent(buf, depth);
+                    push_element(buf, key, &escape_text(&text));
                 }
             }
         }
-    }
-
-    if inner.is_empty() {
-        buf.push_str(&pad);
-        buf.push('<');
-        buf.push_str(field_name);
-        buf.push_str("/>\n");
-    } else {
-        buf.push_str(&pad);
-        buf.push('<');
-        buf.push_str(field_name);
-        buf.push_str(">\n");
-        buf.push_str(&inner);
-        buf.push_str(&pad);
-        buf.push_str("\n");
+        val_dt => {
+            let text = format_field_to_xml(values, i, val_dt, options)?;
+            buf.push('\n');
+            push_indent(buf, depth);
+            push_element(buf, key, &escape_text(&text));
+        }
     }
 
     Ok(())
@@ -701,7 +711,7 @@ fn format_field_to_xml(
                 chrono::NaiveDate::from_num_days_from_ce_opt(days + 719_163).ok_or_else(|| {
                     DataFusionError::Execution(format!("Date32 value out of range: {days}"))
                 })?;
-            Ok(naive.format(&options.date_format).to_string())
+            format_date_xml(naive, &options.date_format)
         }
         DataType::Date64 => {
             let millis = array.as_primitive::().value(row_idx);
@@ -710,7 +720,7 @@ fn format_field_to_xml(
                 .ok_or_else(|| {
                     DataFusionError::Execution(format!("Date64 value out of range: {millis}"))
                 })?;
-            Ok(naive.format(&options.date_format).to_string())
+            format_date_xml(naive, &options.date_format)
         }
         DataType::Decimal128(_, scale) => {
             let raw = array.as_primitive::().value(row_idx);
@@ -723,6 +733,26 @@ fn format_field_to_xml(
     }
 }
 
+fn format_date_xml(date: chrono::NaiveDate, format: &DateTimeFormat) -> Result {
+    use crate::scalar::datetime::format::{DateTimeFormatInput, TimePrecision, TimestampKind};
+
+    let datetime = date.and_hms_opt(0, 0, 0).ok_or_else(|| {
+        DataFusionError::Execution("Failed to create datetime from date".to_string())
+    })?;
+
+    let input = DateTimeFormatInput {
+        datetime,
+        timezone: None,
+        zone_id: None,
+        timestamp_kind: TimestampKind::Normal,
+        precision: TimePrecision::Second,
+    };
+
+    format
+        .format(input)
+        .map_err(|e| DataFusionError::Execution(format!("Failed to format date: {e}")))
+}
+
 fn format_decimal128(raw: i128, scale: u32) -> String {
     if scale == 0 {
         return raw.to_string();
@@ -739,17 +769,73 @@ fn format_decimal128(raw: i128, scale: u32) -> String {
     )
 }
 
-fn format_float(v: f64) -> String {
+/// Formats a `f64` the way Java's `Double.toString` (and therefore Spark) does:
+/// plain decimal for `1e-3 <= |x| < 1e7`, scientific `d.dddEexp` otherwise,
+/// always with at least one fractional digit.
+fn format_double(v: f64) -> String {
+    if v.is_nan() {
+        return "NaN".to_string();
+    }
+    if v.is_infinite() {
+        return if v > 0.0 { "Infinity" } else { "-Infinity" }.to_string();
+    }
+    if v == 0.0 {
+        return "0.0".to_string();
+    }
+    // Rust's `{:e}` yields the shortest round-tripping mantissa in normalized
+    // form `d[.ddd]e`, which carries exactly the significant digits and the
+    // base-10 exponent Java needs.
+    java_float_repr(v < 0.0, &format!("{:e}", v.abs()))
+}
+
+/// Same as [`format_double`] but for `f32` (Spark `FLOAT`). Formatting the `f32`
+/// directly (not via `f64`) keeps the shortest `f32` round-tripping digits.
+fn format_single(v: f32) -> String {
     if v.is_nan() {
-        "NaN".to_string()
-    } else if v.is_infinite() {
-        if v > 0.0 {
-            "Infinity".to_string()
+        return "NaN".to_string();
+    }
+    if v.is_infinite() {
+        return if v > 0.0 { "Infinity" } else { "-Infinity" }.to_string();
+    }
+    if v == 0.0 {
+        return "0.0".to_string();
+    }
+    java_float_repr(v < 0.0, &format!("{:e}", v.abs()))
+}
+
+/// Reformats Rust's normalized `{:e}` output (e.g. `"1.2345678e7"`) into Java
+/// `Double.toString` style. `neg` carries the sign separately.
+fn java_float_repr(neg: bool, rust_sci: &str) -> String {
+    let Some((mantissa, exp_str)) = rust_sci.split_once('e') else {
+        return rust_sci.to_string();
+    };
+    let Ok(sci_exp) = exp_str.parse::() else {
+        return rust_sci.to_string();
+    };
+    let digits: String = mantissa.chars().filter(|c| *c != '.').collect();
+    let sign = if neg { "-" } else { "" };
+
+    if (-3..7).contains(&sci_exp) {
+        // Plain decimal notation.
+        let body = if sci_exp >= 0 {
+            let int_len = sci_exp as usize + 1;
+            if int_len >= digits.len() {
+                format!("{}{}.0", digits, "0".repeat(int_len - digits.len()))
+            } else {
+                format!("{}.{}", &digits[..int_len], &digits[int_len..])
+            }
         } else {
-            "-Infinity".to_string()
-        }
+            format!("0.{}{}", "0".repeat((-sci_exp - 1) as usize), digits)
+        };
+        format!("{sign}{body}")
     } else {
-        format!("{v}")
+        // Scientific notation: one digit before the point, at least one after.
+        let mantissa = if digits.len() == 1 {
+            format!("{}.0", digits)
+        } else {
+            format!("{}.{}", &digits[..1], &digits[1..])
+        };
+        format!("{sign}{mantissa}E{sci_exp}")
     }
 }
 
@@ -764,16 +850,37 @@ fn scalar_to_display_string(scalar: &ScalarValue) -> Result {
         ScalarValue::UInt16(Some(v)) => Ok(v.to_string()),
         ScalarValue::UInt32(Some(v)) => Ok(v.to_string()),
         ScalarValue::UInt64(Some(v)) => Ok(v.to_string()),
-        ScalarValue::Float32(Some(v)) => Ok(format_float(*v as f64)),
-        ScalarValue::Float64(Some(v)) => Ok(format_float(*v)),
+        ScalarValue::Float32(Some(v)) => Ok(format_single(*v)),
+        ScalarValue::Float64(Some(v)) => Ok(format_double(*v)),
         ScalarValue::Utf8(Some(v))
         | ScalarValue::LargeUtf8(Some(v))
         | ScalarValue::Utf8View(Some(v)) => Ok(v.clone()),
+        ScalarValue::Binary(Some(v))
+        | ScalarValue::LargeBinary(Some(v))
+        | ScalarValue::BinaryView(Some(v))
+        | ScalarValue::FixedSizeBinary(_, Some(v)) => Ok(format_binary(v)),
         sv if sv.is_null() => exec_err!("to_xml: scalar_to_display_string called on null: {sv:?}"),
         _ => exec_err!("to_xml: unsupported scalar type for XML serialization: {scalar:?}"),
     }
 }
 
+/// Spark renders binary as space-separated uppercase hex bytes in brackets,
+/// e.g. `X'48656C6C6F'` becomes `[48 65 6C 6C 6F]` and empty binary becomes `[]`.
+fn format_binary(value: &[u8]) -> String {
+    const HEX: &[u8; 16] = b"0123456789ABCDEF";
+    let mut output = String::with_capacity(value.len() * 3 + 2);
+    output.push('[');
+    for (i, byte) in value.iter().enumerate() {
+        if i > 0 {
+            output.push(' ');
+        }
+        output.push(HEX[(byte >> 4) as usize] as char);
+        output.push(HEX[(byte & 0x0f) as usize] as char);
+    }
+    output.push(']');
+    output
+}
+
 fn format_timestamp_field(
     array: &ArrayRef,
     row_idx: usize,
@@ -800,7 +907,7 @@ fn format_timestamp_field(
 
     let secs = micros.div_euclid(1_000_000);
     let nanos = (micros.rem_euclid(1_000_000) * 1_000) as u32;
-    let is_default_format = options.timestamp_ltz_format.is_none();
+    let _is_default_format = options.timestamp_ltz_format.is_none();
 
     if tz_opt.is_some() {
         let tz: Tz = options.session_timezone.parse().map_err(|e| {
@@ -813,20 +920,38 @@ fn format_timestamp_field(
             DataFusionError::Execution(format!("Timestamp out of range: {micros}"))
         })?;
         let local_dt = utc_dt.with_timezone(&tz);
-        if is_default_format {
-            // Default Spark: ISO 8601 with Z suffix for UTC
-            Ok(local_dt
-                .format(SparkToXmlOptions::TIMESTAMP_LTZ_FORMAT_DEFAULT)
-                .to_string()
-                .replace("+00:00", "Z"))
-        } else {
-            let fmt = options.timestamp_ltz_format.as_deref().ok_or_else(|| {
-                DataFusionError::Internal(
-                    "timestamp_ltz_format missing despite non-default path".to_string(),
-                )
-            })?;
 
-            Ok(local_dt.format(fmt).to_string())
+        use chrono::Offset;
+
+        use crate::scalar::datetime::format::{
+            DateTimeFormatInput, TimePrecision, TimeZoneDisplay, TimestampKind,
+        };
+
+        let offset = local_dt.offset().fix();
+        let input = DateTimeFormatInput {
+            datetime: local_dt.naive_local(),
+            timezone: Some(TimeZoneDisplay {
+                offset,
+                name: Some(&options.session_timezone),
+            }),
+            zone_id: Some(&options.session_timezone),
+            timestamp_kind: TimestampKind::Normal,
+            precision: TimePrecision::Microsecond,
+        };
+
+        if let Some(ref fmt) = options.timestamp_ltz_format {
+            Ok(fmt.format(input).map_err(|e| {
+                DataFusionError::Execution(format!("Failed to format timestamp: {e}"))
+            })?)
+        } else {
+            // Use default format
+            #[expect(clippy::expect_used)]
+            let default_fmt =
+                DateTimeFormat::for_formatting(SparkToXmlOptions::TIMESTAMP_LTZ_FORMAT_DEFAULT)
+                    .expect("default timestamp LTZ format should be valid");
+            Ok(default_fmt.format(input).map_err(|e| {
+                DataFusionError::Execution(format!("Failed to format timestamp: {e}"))
+            })?)
         }
     } else {
         let naive = DateTime::from_timestamp(secs, nanos)
@@ -834,32 +959,47 @@ fn format_timestamp_field(
             .ok_or_else(|| {
                 DataFusionError::Execution(format!("Timestamp out of range: {micros}"))
             })?;
-        Ok(naive.format(&options.timestamp_ntz_format).to_string())
+
+        use crate::scalar::datetime::format::{DateTimeFormatInput, TimePrecision, TimestampKind};
+
+        let input = DateTimeFormatInput {
+            datetime: naive,
+            timezone: None,
+            zone_id: None,
+            timestamp_kind: TimestampKind::Normal,
+            precision: TimePrecision::Microsecond,
+        };
+
+        Ok(options
+            .timestamp_ntz_format
+            .format(input)
+            .map_err(|e| DataFusionError::Execution(format!("Failed to format timestamp: {e}")))?)
     }
 }
 
+/// Spark's `StaxXmlGenerator` escapes only `&` and `<` in element text; `>` is
+/// left literal.
 fn escape_text(s: &str) -> String {
     let mut out = String::with_capacity(s.len());
     for c in s.chars() {
         match c {
             '&' => out.push_str("&"),
             '<' => out.push_str("<"),
-            '>' => out.push_str(">"),
             _ => out.push(c),
         }
     }
     out
 }
 
+/// In attribute values Spark escapes `&`, `<` and `"`; `>` and `'` are left
+/// literal.
 fn escape_attr(s: &str) -> String {
     let mut out = String::with_capacity(s.len());
     for c in s.chars() {
         match c {
             '&' => out.push_str("&"),
             '<' => out.push_str("<"),
-            '>' => out.push_str(">"),
             '"' => out.push_str("""),
-            '\'' => out.push_str("'"),
             _ => out.push(c),
         }
     }
@@ -936,14 +1076,26 @@ mod tests {
 
     #[test]
     fn test_escape_text() {
-        assert_eq!(escape_text("a & b < c > d"), "a & b < c > d");
+        // Spark escapes only `&` and `<`; `>` stays literal.
+        assert_eq!(escape_text("a & b < c > d"), "a & b < c > d");
         assert_eq!(escape_text("clean"), "clean");
     }
 
     #[test]
     fn test_escape_attr() {
+        // Spark escapes `&`, `<` and `"` in attributes; `>` and `'` stay literal.
         assert_eq!(escape_attr(r#"say "hi""#), "say "hi"");
-        assert_eq!(escape_attr("it's"), "it's");
+        assert_eq!(escape_attr("it's > that"), "it's > that");
+    }
+
+    #[test]
+    fn test_format_binary() {
+        assert_eq!(
+            format_binary(&[0x48, 0x65, 0x6C, 0x6C, 0x6F]),
+            "[48 65 6C 6C 6F]"
+        );
+        assert_eq!(format_binary(&[]), "[]");
+        assert_eq!(format_binary(&[0x00, 0xFF]), "[00 FF]");
     }
 
     #[test]
@@ -958,12 +1110,36 @@ mod tests {
     }
 
     #[test]
-    fn test_format_float() -> Result<(), Box> {
-        assert_eq!(format_float(f64::NAN), "NaN");
-        assert_eq!(format_float(f64::INFINITY), "Infinity");
-        assert_eq!(format_float(f64::NEG_INFINITY), "-Infinity");
-        assert_eq!(format_float(3.44), "3.44");
-        assert_eq!(format_float(0.0), "0");
-        Ok(())
+    fn test_format_double() {
+        // Values pinned against Java `Double.toString` (Spark JVM ground truth).
+        assert_eq!(format_double(f64::NAN), "NaN");
+        assert_eq!(format_double(f64::INFINITY), "Infinity");
+        assert_eq!(format_double(f64::NEG_INFINITY), "-Infinity");
+        assert_eq!(format_double(0.0), "0.0");
+        assert_eq!(format_double(-0.0), "0.0");
+        assert_eq!(format_double(1.0), "1.0");
+        assert_eq!(format_double(100.0), "100.0");
+        assert_eq!(format_double(2.71), "2.71");
+        assert_eq!(format_double(-123.456), "-123.456");
+        assert_eq!(format_double(0.001), "0.001");
+        assert_eq!(format_double(0.0009), "9.0E-4");
+        assert_eq!(format_double(0.0001), "1.0E-4");
+        assert_eq!(format_double(9999999.0), "9999999.0");
+        assert_eq!(format_double(10000000.0), "1.0E7");
+        assert_eq!(format_double(12345678.0), "1.2345678E7");
+        assert_eq!(format_double(1e20), "1.0E20");
+        assert_eq!(format_double(1e-7), "1.0E-7");
+        assert_eq!(format_double(1e308), "1.0E308");
+        assert_eq!(format_double(123456789012345.0), "1.23456789012345E14");
+    }
+
+    #[test]
+    fn test_format_single() {
+        assert_eq!(format_single(f32::NAN), "NaN");
+        assert_eq!(format_single(f32::INFINITY), "Infinity");
+        assert_eq!(format_single(1.0), "1.0");
+        assert_eq!(format_single(0.1), "0.1");
+        assert_eq!(format_single(123.456), "123.456");
+        assert_eq!(format_single(10000000.0), "1.0E7");
     }
 }
diff --git a/crates/sail-function/src/scalar/xml/xpath.rs b/crates/sail-function/src/scalar/xml/xpath.rs
index 4c9ed33a6..5d064fe74 100644
--- a/crates/sail-function/src/scalar/xml/xpath.rs
+++ b/crates/sail-function/src/scalar/xml/xpath.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::array::{Array, ArrayRef, ListBuilder, StringArray, StringBuilder};
 use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion_common::utils::take_function_args;
-use datafusion_common::{plan_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use xee_xpath::{Documents, Item, Queries, Query};
 
diff --git a/crates/sail-function/src/scalar/xml/xpath_typed.rs b/crates/sail-function/src/scalar/xml/xpath_typed.rs
index 8d00bc191..f49124107 100644
--- a/crates/sail-function/src/scalar/xml/xpath_typed.rs
+++ b/crates/sail-function/src/scalar/xml/xpath_typed.rs
@@ -6,7 +6,7 @@ use datafusion::arrow::array::{
 };
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::utils::take_function_args;
-use datafusion_common::{plan_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, plan_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use xee_xpath::{Documents, Queries, Query};
 
diff --git a/crates/sail-function/src/schema_inference.rs b/crates/sail-function/src/schema_inference.rs
new file mode 100644
index 000000000..5e163a04e
--- /dev/null
+++ b/crates/sail-function/src/schema_inference.rs
@@ -0,0 +1,78 @@
+/// A recursively inferred schema before it is rendered as a Spark type string.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) enum InferredType {
+    Null,
+    Boolean,
+    Long,
+    Float,
+    Decimal(u8, u8),
+    Double,
+    String,
+    Binary,
+    Date,
+    Timestamp,
+    TimestampNtz,
+    Array(Box),
+    Struct(Vec<(String, InferredType)>),
+    Variant,
+}
+
+/// Defines how format-specific incompatible scalar types are merged.
+pub(crate) trait TypeMerger {
+    fn merge_atomic(&self, left: InferredType, right: InferredType) -> InferredType;
+}
+
+impl InferredType {
+    /// Recursively merges two inferred types while delegating format-specific
+    /// scalar coercion and incompatible-type handling to the merger.
+    pub(crate) fn merge_with(self, other: InferredType, merger: &impl TypeMerger) -> InferredType {
+        match (self, other) {
+            (InferredType::Null, other) | (other, InferredType::Null) => other,
+            (left, right) if left == right => left,
+            (InferredType::Array(left), InferredType::Array(right)) => {
+                InferredType::Array(Box::new(left.merge_with(*right, merger)))
+            }
+            (InferredType::Struct(left), InferredType::Struct(right)) => {
+                InferredType::Struct(merge_fields(left, right, merger))
+            }
+            (left, right) => merger.merge_atomic(left, right),
+        }
+    }
+
+    pub(crate) fn estimated_size(&self) -> usize {
+        match self {
+            InferredType::Array(element) => {
+                std::mem::size_of::() + element.estimated_size()
+            }
+            InferredType::Struct(fields) => fields
+                .iter()
+                .map(|(name, ty)| {
+                    name.len() + ty.estimated_size() + std::mem::size_of::<(String, InferredType)>()
+                })
+                .sum(),
+            _ => 0,
+        }
+    }
+}
+
+fn merge_fields(
+    left: Vec<(String, InferredType)>,
+    right: Vec<(String, InferredType)>,
+    merger: &impl TypeMerger,
+) -> Vec<(String, InferredType)> {
+    let mut fields = left;
+    fields.extend(right);
+    fields.sort_by(|(left, _), (right, _)| left.cmp(right));
+
+    let mut merged: Vec<(String, InferredType)> = Vec::new();
+    for (name, ty) in fields {
+        match merged.last_mut() {
+            Some((last_name, last_type)) if *last_name == name => {
+                let previous = std::mem::replace(last_type, InferredType::Null);
+                *last_type = previous.merge_with(ty, merger);
+            }
+            _ => merged.push((name, ty)),
+        }
+    }
+    merged
+}
diff --git a/crates/sail-function/src/theta_sketch.rs b/crates/sail-function/src/theta_sketch.rs
index cfc853e8b..87d21d1ad 100644
--- a/crates/sail-function/src/theta_sketch.rs
+++ b/crates/sail-function/src/theta_sketch.rs
@@ -1,11 +1,10 @@
 use std::collections::{BTreeSet, HashSet};
 
-use arrow::array::{
-    Array, ArrayRef, BinaryArray, Float32Array, Float64Array, Int32Array, Int64Array,
-    LargeListArray, LargeStringArray, ListArray, StringArray, StringViewArray, UInt32Array,
-    UInt64Array,
+use arrow::array::{Array, ArrayRef, AsArray};
+use arrow::datatypes::{
+    DataType, Float32Type, Float64Type, Int32Type, Int64Type, UInt32Type, UInt64Type,
 };
-use datafusion_common::{exec_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, exec_err};
 use datasketches::hash_value::{canonical_float, raw_bytes};
 use datasketches::theta::{CompactThetaSketch, ThetaSketch};
 
@@ -59,8 +58,83 @@ pub(crate) fn normalize_sketch_bytes(bytes: &[u8], function_name: &str) -> Resul
 }
 
 pub(crate) fn update_sketch_from_array(sketch: &mut ThetaSketch, values: &ArrayRef) -> Result<()> {
-    for row in 0..values.len() {
-        update_sketch_from_array_value(sketch, values.as_ref(), row)?;
+    // Downcast once outside the row loop (the data type and concrete array are
+    // loop-invariant); `iter().flatten()` skips nulls without a per-row branch.
+    match values.data_type() {
+        DataType::Int32 => {
+            for v in values.as_primitive::().iter().flatten() {
+                sketch.update(v as i64);
+            }
+        }
+        DataType::Int64 => {
+            for v in values.as_primitive::().iter().flatten() {
+                sketch.update(v);
+            }
+        }
+        DataType::UInt32 => {
+            for v in values.as_primitive::().iter().flatten() {
+                sketch.update(v as u64);
+            }
+        }
+        DataType::UInt64 => {
+            for v in values.as_primitive::().iter().flatten() {
+                sketch.update(v);
+            }
+        }
+        DataType::Float32 => {
+            for v in values.as_primitive::().iter().flatten() {
+                sketch.update(canonical_float::from_f32(v));
+            }
+        }
+        DataType::Float64 => {
+            for v in values.as_primitive::().iter().flatten() {
+                sketch.update(canonical_float::from_f64(v));
+            }
+        }
+        DataType::Binary => {
+            for v in values.as_binary::().iter().flatten() {
+                if !v.is_empty() {
+                    sketch.update(raw_bytes::from_slice(v));
+                }
+            }
+        }
+        DataType::Utf8 => {
+            for v in values.as_string::().iter().flatten() {
+                if !v.is_empty() {
+                    sketch.update(raw_bytes::from_str(v));
+                }
+            }
+        }
+        DataType::LargeUtf8 => {
+            for v in values.as_string::().iter().flatten() {
+                if !v.is_empty() {
+                    sketch.update(raw_bytes::from_str(v));
+                }
+            }
+        }
+        DataType::Utf8View => {
+            for v in values.as_string_view().iter().flatten() {
+                if !v.is_empty() {
+                    sketch.update(raw_bytes::from_str(v));
+                }
+            }
+        }
+        DataType::List(_) => {
+            let mut scratch = Vec::new();
+            for cell in values.as_list::().iter().flatten() {
+                update_sketch_from_list(sketch, &cell, &mut scratch)?;
+            }
+        }
+        DataType::LargeList(_) => {
+            let mut scratch = Vec::new();
+            for cell in values.as_list::().iter().flatten() {
+                update_sketch_from_list(sketch, &cell, &mut scratch)?;
+            }
+        }
+        DataType::Null => {}
+        data_type => {
+            return exec_err!("theta_sketch_agg does not support input type {data_type}");
+        }
     }
     Ok(())
 }
@@ -125,10 +199,10 @@ where
         }
     }
 
-    if entries.len() > nominal_entries {
-        if let Some(next_theta) = entries.pop_last() {
-            theta = theta.min(next_theta);
-        }
+    if entries.len() > nominal_entries
+        && let Some(next_theta) = entries.pop_last()
+    {
+        theta = theta.min(next_theta);
     }
     let entries: Vec = entries.into_iter().collect();
     let empty = !saw_non_empty && entries.is_empty() && theta == MAX_THETA;
@@ -180,186 +254,46 @@ pub(crate) fn difference_sketch_bytes(
     serialize_compact_sketch(entries, theta, output_seed_hash(&left, &right), empty)
 }
 
-fn update_sketch_from_array_value(
+fn update_sketch_from_list(
     sketch: &mut ThetaSketch,
-    values: &dyn Array,
-    row: usize,
+    values: &ArrayRef,
+    scratch: &mut Vec,
 ) -> Result<()> {
-    if values.is_null(row) {
-        return Ok(());
-    }
-
-    match values.data_type() {
-        arrow::datatypes::DataType::Int32 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected Int32Array".to_string())
-                })?;
-            sketch.update(values.value(row) as i64);
-        }
-        arrow::datatypes::DataType::Int64 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected Int64Array".to_string())
-                })?;
-            sketch.update(values.value(row));
-        }
-        arrow::datatypes::DataType::UInt32 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected UInt32Array".to_string())
-                })?;
-            sketch.update(values.value(row) as u64);
-        }
-        arrow::datatypes::DataType::UInt64 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected UInt64Array".to_string())
-                })?;
-            sketch.update(values.value(row));
-        }
-        arrow::datatypes::DataType::Float32 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected Float32Array".to_string())
-                })?;
-            sketch.update(canonical_float::from_f32(values.value(row)));
-        }
-        arrow::datatypes::DataType::Float64 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected Float64Array".to_string())
-                })?;
-            sketch.update(canonical_float::from_f64(values.value(row)));
-        }
-        arrow::datatypes::DataType::Binary => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected BinaryArray".to_string())
-                })?;
-            let value = values.value(row);
-            if !value.is_empty() {
-                sketch.update(raw_bytes::from_slice(value));
-            }
-        }
-        arrow::datatypes::DataType::Utf8 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected StringArray".to_string())
-                })?;
-            let value = values.value(row);
-            if !value.is_empty() {
-                sketch.update(raw_bytes::from_str(value));
-            }
-        }
-        arrow::datatypes::DataType::LargeUtf8 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected LargeStringArray".to_string())
-                })?;
-            let value = values.value(row);
-            if !value.is_empty() {
-                sketch.update(raw_bytes::from_str(value));
-            }
-        }
-        arrow::datatypes::DataType::Utf8View => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected StringViewArray".to_string())
-                })?;
-            let value = values.value(row);
-            if !value.is_empty() {
-                sketch.update(raw_bytes::from_str(value));
-            }
-        }
-        arrow::datatypes::DataType::List(_) => {
-            let values = values.as_any().downcast_ref::().ok_or_else(|| {
-                DataFusionError::Internal("theta sketch expected ListArray".to_string())
-            })?;
-            update_sketch_from_list(sketch, &values.value(row))?;
-        }
-        arrow::datatypes::DataType::LargeList(_) => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal("theta sketch expected LargeListArray".to_string())
-                })?;
-            update_sketch_from_list(sketch, &values.value(row))?;
-        }
-        arrow::datatypes::DataType::Null => {}
-        data_type => {
-            return exec_err!("theta_sketch_agg does not support input type {data_type}");
-        }
-    }
-    Ok(())
-}
-
-fn update_sketch_from_list(sketch: &mut ThetaSketch, values: &ArrayRef) -> Result<()> {
     if values.is_empty() {
         return Ok(());
     }
 
+    // Spark hashes the whole array as one entity, so the element bytes must be
+    // concatenated into a contiguous buffer. `scratch` is reused across list cells
+    // (cleared per cell) to avoid a fresh allocation for every array value.
     match values.data_type() {
-        arrow::datatypes::DataType::Int32 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal(
-                        "theta sketch expected Int32Array list values".to_string(),
-                    )
-                })?;
-            let mut bytes = Vec::with_capacity(values.len() * std::mem::size_of::());
+        DataType::Int32 => {
+            let values = values.as_primitive::();
+            scratch.clear();
+            scratch.reserve(values.len() * std::mem::size_of::());
             for row in 0..values.len() {
                 let value = if values.is_null(row) {
                     0
                 } else {
                     values.value(row)
                 };
-                bytes.extend_from_slice(&value.to_le_bytes());
+                scratch.extend_from_slice(&value.to_le_bytes());
             }
-            sketch.update(raw_bytes::from_slice(&bytes));
+            sketch.update(raw_bytes::from_slice(scratch.as_slice()));
         }
-        arrow::datatypes::DataType::Int64 => {
-            let values = values
-                .as_any()
-                .downcast_ref::()
-                .ok_or_else(|| {
-                    DataFusionError::Internal(
-                        "theta sketch expected Int64Array list values".to_string(),
-                    )
-                })?;
-            let mut bytes = Vec::with_capacity(values.len() * std::mem::size_of::());
+        DataType::Int64 => {
+            let values = values.as_primitive::();
+            scratch.clear();
+            scratch.reserve(values.len() * std::mem::size_of::());
             for row in 0..values.len() {
                 let value = if values.is_null(row) {
                     0
                 } else {
                     values.value(row)
                 };
-                bytes.extend_from_slice(&value.to_le_bytes());
+                scratch.extend_from_slice(&value.to_le_bytes());
             }
-            sketch.update(raw_bytes::from_slice(&bytes));
+            sketch.update(raw_bytes::from_slice(scratch.as_slice()));
         }
         data_type => {
             return exec_err!("theta_sketch_agg does not support array element type {data_type}");
diff --git a/crates/sail-function/src/window/mod.rs b/crates/sail-function/src/window/mod.rs
index ff570f1eb..4fbd7cbf2 100644
--- a/crates/sail-function/src/window/mod.rs
+++ b/crates/sail-function/src/window/mod.rs
@@ -1,3 +1,7 @@
+mod spark_first_last_value;
 mod spark_ntile;
 
-pub use spark_ntile::{spark_ntile_udwf, SparkNtile};
+pub use spark_first_last_value::{
+    SparkFirstLastValue, SparkFirstLastValueKind, spark_first_value_udwf, spark_last_value_udwf,
+};
+pub use spark_ntile::{SparkNtile, spark_ntile_udwf};
diff --git a/crates/sail-function/src/window/spark_first_last_value.rs b/crates/sail-function/src/window/spark_first_last_value.rs
new file mode 100644
index 000000000..4386c5c1b
--- /dev/null
+++ b/crates/sail-function/src/window/spark_first_last_value.rs
@@ -0,0 +1,170 @@
+use std::fmt::Debug;
+use std::ops::Range;
+use std::sync::Arc;
+
+use datafusion::arrow::array::ArrayRef;
+use datafusion::arrow::buffer::NullBuffer;
+use datafusion::arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion::error::Result;
+use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs};
+use datafusion::logical_expr::{
+    LimitEffect, PartitionEvaluator, ReversedUDWF, Signature, TypeSignature, Volatility, WindowUDF,
+    WindowUDFImpl,
+};
+use datafusion::scalar::ScalarValue;
+
+pub fn spark_first_value_udwf(ignore_nulls: bool) -> Arc {
+    Arc::new(WindowUDF::from(SparkFirstLastValue::first(ignore_nulls)))
+}
+
+pub fn spark_last_value_udwf(ignore_nulls: bool) -> Arc {
+    Arc::new(WindowUDF::from(SparkFirstLastValue::last(ignore_nulls)))
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub enum SparkFirstLastValueKind {
+    First,
+    Last,
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkFirstLastValue {
+    kind: SparkFirstLastValueKind,
+    ignore_nulls: bool,
+    signature: Signature,
+}
+
+impl SparkFirstLastValue {
+    pub fn first(ignore_nulls: bool) -> Self {
+        Self::new(SparkFirstLastValueKind::First, ignore_nulls)
+    }
+
+    pub fn last(ignore_nulls: bool) -> Self {
+        Self::new(SparkFirstLastValueKind::Last, ignore_nulls)
+    }
+
+    pub fn kind(&self) -> SparkFirstLastValueKind {
+        self.kind
+    }
+
+    pub fn ignore_nulls(&self) -> bool {
+        self.ignore_nulls
+    }
+
+    fn new(kind: SparkFirstLastValueKind, ignore_nulls: bool) -> Self {
+        Self {
+            kind,
+            ignore_nulls,
+            signature: Signature::one_of(
+                vec![TypeSignature::Nullary, TypeSignature::Any(1)],
+                Volatility::Immutable,
+            ),
+        }
+    }
+}
+
+impl WindowUDFImpl for SparkFirstLastValue {
+    fn name(&self) -> &str {
+        match self.kind {
+            SparkFirstLastValueKind::First => "first_value",
+            SparkFirstLastValueKind::Last => "last_value",
+        }
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn partition_evaluator(
+        &self,
+        _partition_evaluator_args: PartitionEvaluatorArgs,
+    ) -> Result> {
+        Ok(Box::new(SparkFirstLastValueEvaluator {
+            kind: self.kind,
+            ignore_nulls: self.ignore_nulls,
+        }))
+    }
+
+    fn field(&self, field_args: WindowUDFFieldArgs) -> Result {
+        let input_field = field_args
+            .input_fields()
+            .first()
+            .cloned()
+            .unwrap_or_else(|| Arc::new(Field::new(field_args.name(), DataType::Null, true)));
+        Ok(input_field
+            .as_ref()
+            .clone()
+            .with_name(field_args.name())
+            .with_nullable(true)
+            .into())
+    }
+
+    fn reverse_expr(&self) -> ReversedUDWF {
+        match self.kind {
+            SparkFirstLastValueKind::First => {
+                ReversedUDWF::Reversed(spark_last_value_udwf(self.ignore_nulls))
+            }
+            SparkFirstLastValueKind::Last => {
+                ReversedUDWF::Reversed(spark_first_value_udwf(self.ignore_nulls))
+            }
+        }
+    }
+
+    fn limit_effect(
+        &self,
+        _args: &[Arc],
+    ) -> LimitEffect {
+        LimitEffect::None
+    }
+}
+
+#[derive(Debug)]
+struct SparkFirstLastValueEvaluator {
+    kind: SparkFirstLastValueKind,
+    ignore_nulls: bool,
+}
+
+impl PartitionEvaluator for SparkFirstLastValueEvaluator {
+    fn evaluate(&mut self, values: &[ArrayRef], range: &Range) -> Result {
+        let array = &values[0];
+        if range.start == range.end {
+            return ScalarValue::try_from(array.data_type());
+        }
+        match self.valid_index(array, range) {
+            Some(index) => ScalarValue::try_from_array(array, index),
+            None => ScalarValue::try_from(array.data_type()),
+        }
+    }
+
+    fn supports_bounded_execution(&self) -> bool {
+        true
+    }
+
+    fn uses_window_frame(&self) -> bool {
+        true
+    }
+}
+
+impl SparkFirstLastValueEvaluator {
+    fn valid_index(&self, array: &ArrayRef, range: &Range) -> Option {
+        if self.ignore_nulls {
+            let slice = array.slice(range.start, range.end - range.start);
+            if let Some(nulls) = slice.nulls()
+                && nulls.null_count() > 0
+            {
+                return self.valid_index_with_nulls(nulls, range.start);
+            }
+        }
+        match self.kind {
+            SparkFirstLastValueKind::First => Some(range.start),
+            SparkFirstLastValueKind::Last => Some(range.end - 1),
+        }
+    }
+
+    fn valid_index_with_nulls(&self, nulls: &NullBuffer, offset: usize) -> Option {
+        match self.kind {
+            SparkFirstLastValueKind::First => nulls.valid_indices().next().map(|idx| idx + offset),
+            SparkFirstLastValueKind::Last => nulls.valid_indices().last().map(|idx| idx + offset),
+        }
+    }
+}
diff --git a/crates/sail-function/src/window/spark_ntile.rs b/crates/sail-function/src/window/spark_ntile.rs
index d5b48fa40..fb559fda7 100644
--- a/crates/sail-function/src/window/spark_ntile.rs
+++ b/crates/sail-function/src/window/spark_ntile.rs
@@ -141,10 +141,10 @@ fn get_scalar_value_from_args(
     input_exprs: &[Arc],
     index: usize,
 ) -> Result> {
-    if let Some(expr) = input_exprs.get(index) {
-        if let Some(literal) = expr.downcast_ref::() {
-            return Ok(Some(literal.value().clone()));
-        }
+    if let Some(expr) = input_exprs.get(index)
+        && let Some(literal) = expr.downcast_ref::()
+    {
+        return Ok(Some(literal.value().clone()));
     }
     Ok(None)
 }
diff --git a/crates/sail-iceberg/Cargo.toml b/crates/sail-iceberg/Cargo.toml
index 6fa615ac6..fbdd2568e 100644
--- a/crates/sail-iceberg/Cargo.toml
+++ b/crates/sail-iceberg/Cargo.toml
@@ -20,6 +20,8 @@ sail-catalog = { path = "../sail-catalog" }
 sail-common = { path = "../sail-common" }
 sail-common-datafusion = { path = "../sail-common-datafusion" }
 sail-data-source = { path = "../sail-data-source" }
+sail-logical-plan = { path = "../sail-logical-plan" }
+sail-physical-plan = { path = "../sail-physical-plan" }
 sail-sql-analyzer = { path = "../sail-sql-analyzer" }
 
 # DataFusion dependencies
diff --git a/crates/sail-iceberg/src/catalog_support/commit.rs b/crates/sail-iceberg/src/catalog_support/commit.rs
index 60c6e6efb..04935bed7 100644
--- a/crates/sail-iceberg/src/catalog_support/commit.rs
+++ b/crates/sail-iceberg/src/catalog_support/commit.rs
@@ -4,8 +4,8 @@ use sail_catalog::lakehouse::{LakehouseCommitOutcome, LakehouseCommitRequest};
 use sail_catalog::manager::CatalogManager;
 use sail_catalog::provider::AlterTableOptions;
 use sail_common_datafusion::catalog::managed::{
-    existing_metadata_location_key, METADATA_LOCATION_UNDERSCORE_KEY,
-    PREVIOUS_METADATA_LOCATION_KEY,
+    METADATA_LOCATION_UNDERSCORE_KEY, PREVIOUS_METADATA_LOCATION_KEY,
+    existing_metadata_location_key,
 };
 use sail_common_datafusion::catalog::{
     CommitAuthority, LakehouseExecutionContext, TableKind, TableStatus,
@@ -279,14 +279,13 @@ pub(crate) fn catalog_requirements(
     );
     requirements.extend_from_slice(commit_requirements);
     requirements.extend_from_slice(action_requirements);
-    if let Some(uuid) = table_meta.table_uuid {
-        if !requirements
+    if let Some(uuid) = table_meta.table_uuid
+        && !requirements
             .iter()
             .any(|requirement| matches!(requirement, TableRequirement::UuidMatch { uuid: u } if *u == uuid))
         {
             requirements.push(TableRequirement::UuidMatch { uuid });
         }
-    }
     requirements
 }
 
diff --git a/crates/sail-iceberg/src/datasource/provider.rs b/crates/sail-iceberg/src/datasource/provider.rs
index 659c08745..72b4c791e 100644
--- a/crates/sail-iceberg/src/datasource/provider.rs
+++ b/crates/sail-iceberg/src/datasource/provider.rs
@@ -15,30 +15,31 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use datafusion::arrow::datatypes::Schema as ArrowSchema;
-use datafusion::catalog::memory::DataSourceExec;
+use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
 use datafusion::catalog::Session;
+use datafusion::catalog::memory::DataSourceExec;
 use datafusion::common::scalar::ScalarValue;
 use datafusion::common::stats::{ColumnStatistics, Precision, Statistics};
-use datafusion::common::{plan_err, Result, ToDFSchema};
+use datafusion::common::{Result, ToDFSchema, plan_err};
 use datafusion::config::TableParquetOptions;
 use datafusion::datasource::listing::PartitionedFile;
 use datafusion::datasource::physical_plan::{FileGroup, FileScanConfigBuilder, ParquetSource};
+use datafusion::datasource::table_schema::TableSchema;
 use datafusion::datasource::{TableProvider, TableType};
 use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::logical_expr::utils::conjunction;
 use datafusion::logical_expr::{
     BinaryExpr, Expr, LogicalPlan, Operator, TableProviderFilterPushDown,
 };
-use datafusion::physical_expr::expressions::Column;
 use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr::expressions::Column;
 use datafusion::physical_expr_adapter::PhysicalExprAdapterFactory;
+use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_plan::empty::EmptyExec;
 use datafusion::physical_plan::filter::FilterExec;
 use datafusion::physical_plan::limit::GlobalLimitExec;
 use datafusion::physical_plan::projection::ProjectionExec;
 use datafusion::physical_plan::union::UnionExec;
-use datafusion::physical_plan::ExecutionPlan;
 use object_store::ObjectMeta;
 use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapterFactory;
 use url::Url;
@@ -49,11 +50,15 @@ use crate::datasource::pruning::{
 };
 use crate::datasource::type_converter::iceberg_schema_to_arrow;
 use crate::io::{
-    load_manifest as io_load_manifest, load_manifest_list as io_load_manifest_list, StoreContext,
+    StoreContext, load_manifest as io_load_manifest, load_manifest_list as io_load_manifest_list,
 };
 use crate::physical_plan::delete_apply_exec::IcebergDeleteApplyExec;
 use crate::physical_plan::discovery_exec::IcebergDiscoveryExec;
 use crate::physical_plan::manifest_scan_exec::IcebergManifestScanExec;
+use crate::physical_plan::merge_metadata_exec::IcebergMergeMetadataExec;
+use crate::row_level_metadata::{
+    MERGE_PARTITION_COLUMN, MERGE_PARTITION_SPEC_ID_COLUMN, RowLevelMetadataColumns,
+};
 use crate::spec::delete_index::{DeleteFileIndex, DeleteFileRef};
 use crate::spec::transform::Transform;
 use crate::spec::types::values::Literal;
@@ -64,7 +69,7 @@ use crate::utils::conversions::primitive_to_scalar_default;
 use crate::utils::get_object_store_from_session;
 
 /// Iceberg table provider for DataFusion
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct IcebergTableProvider {
     /// The table location (URI)
     table_uri: String,
@@ -79,6 +84,12 @@ pub struct IcebergTableProvider {
     default_spec_id: i32,
     /// Arrow schema for DataFusion
     arrow_schema: Arc,
+    /// Output schema exposed to DataFusion; may include MERGE metadata columns.
+    output_schema: Arc,
+    /// Optional file-path metadata column for row-level write planning.
+    file_column_name: Option,
+    /// Optional file-local row-index metadata column for row-level write planning.
+    row_index_column_name: Option,
     /// Whether to use the metadata-as-data read path (lazy manifest scanning)
     metadata_as_data_read: bool,
 }
@@ -117,7 +128,10 @@ impl IcebergTableProvider {
             snapshot: Some(snapshot),
             partition_specs,
             default_spec_id,
+            output_schema: arrow_schema.clone(),
             arrow_schema,
+            file_column_name: None,
+            row_index_column_name: None,
             metadata_as_data_read: false,
         })
     }
@@ -150,7 +164,10 @@ impl IcebergTableProvider {
             snapshot: None,
             partition_specs,
             default_spec_id,
+            output_schema: arrow_schema.clone(),
             arrow_schema,
+            file_column_name: None,
+            row_index_column_name: None,
             metadata_as_data_read: false,
         })
     }
@@ -161,6 +178,41 @@ impl IcebergTableProvider {
         self
     }
 
+    pub fn file_column_name(&self) -> Option<&str> {
+        self.file_column_name.as_deref()
+    }
+
+    pub fn row_index_column_name(&self) -> Option<&str> {
+        self.row_index_column_name.as_deref()
+    }
+
+    pub fn with_file_column(mut self, name: &str) -> Result {
+        self.file_column_name = Some(name.to_string());
+        self.rebuild_output_schema()?;
+        Ok(self)
+    }
+
+    pub fn with_row_index_column(mut self, name: &str) -> Result {
+        self.row_index_column_name = Some(name.to_string());
+        self.rebuild_output_schema()?;
+        Ok(self)
+    }
+
+    fn rebuild_output_schema(&mut self) -> Result<()> {
+        let metadata_columns = RowLevelMetadataColumns::new(
+            self.file_column_name.as_deref(),
+            self.row_index_column_name.as_deref(),
+        );
+        let metadata_columns = if self.file_column_name.is_some() {
+            metadata_columns.with_delete_file_metadata()
+        } else {
+            metadata_columns
+        };
+        self.output_schema =
+            Arc::new(metadata_columns.append_to_schema(self.arrow_schema.as_ref())?);
+        Ok(())
+    }
+
     fn reorder_arrow_schema_for_identity_partitions(
         schema: &Schema,
         partition_specs: &[PartitionSpec],
@@ -186,10 +238,10 @@ impl IcebergTableProvider {
 
         let mut identity_cols: Vec = Vec::new();
         for pf in spec.fields().iter() {
-            if matches!(pf.transform, Transform::Identity) {
-                if let Some(field) = schema.field_by_id(pf.source_id) {
-                    identity_cols.push(field.name.clone());
-                }
+            if matches!(pf.transform, Transform::Identity)
+                && let Some(field) = schema.field_by_id(pf.source_id)
+            {
+                identity_cols.push(field.name.clone());
             }
         }
         if identity_cols.is_empty() {
@@ -236,11 +288,11 @@ impl IcebergTableProvider {
             Some(projection) => {
                 let fields = projection
                     .iter()
-                    .map(|idx| self.arrow_schema.field(*idx).clone())
+                    .map(|idx| self.output_schema.field(*idx).clone())
                     .collect::>();
                 Ok(Arc::new(ArrowSchema::new(fields)))
             }
-            None => Ok(self.arrow_schema.clone()),
+            None => Ok(self.output_schema.clone()),
         }
     }
 
@@ -466,6 +518,36 @@ impl IcebergTableProvider {
         Ok(partitioned_files)
     }
 
+    fn create_merge_partitioned_files(
+        &self,
+        store_ctx: &StoreContext,
+        data_files: Vec,
+    ) -> Result> {
+        let file_metadata = data_files
+            .iter()
+            .map(|file| {
+                Ok((
+                    file.file_path.clone(),
+                    file.partition_spec_id,
+                    serde_json::to_string(&file.partition).map_err(|error| {
+                        datafusion::common::DataFusionError::External(Box::new(error))
+                    })?,
+                ))
+            })
+            .collect::>>()?;
+        let mut partitioned_files = self.create_partitioned_files(store_ctx, data_files)?;
+        for (partitioned_file, (file_path, partition_spec_id, partition_json)) in
+            partitioned_files.iter_mut().zip(file_metadata)
+        {
+            partitioned_file.partition_values = vec![
+                ScalarValue::Utf8(Some(file_path)),
+                ScalarValue::Int32(Some(partition_spec_id)),
+                ScalarValue::Utf8(Some(partition_json)),
+            ];
+        }
+        Ok(partitioned_files)
+    }
+
     /// Create file groups from partitioned files
     fn create_file_groups(&self, partitioned_files: Vec) -> Vec {
         // Group files by partition values
@@ -516,6 +598,30 @@ impl IcebergTableProvider {
         Ok(Arc::new(parquet_source))
     }
 
+    fn build_merge_parquet_source(
+        &self,
+        session: &dyn Session,
+        file_column_name: &str,
+    ) -> Arc {
+        let parquet_options = TableParquetOptions {
+            global: session.config().options().execution.parquet.clone(),
+            ..Default::default()
+        };
+        let table_schema = TableSchema::new(
+            self.arrow_schema.clone(),
+            vec![
+                Arc::new(Field::new(file_column_name, DataType::Utf8, false)),
+                Arc::new(Field::new(
+                    MERGE_PARTITION_SPEC_ID_COLUMN,
+                    DataType::Int32,
+                    false,
+                )),
+                Arc::new(Field::new(MERGE_PARTITION_COLUMN, DataType::Utf8, false)),
+            ],
+        );
+        Arc::new(ParquetSource::new(table_schema).with_table_parquet_options(parquet_options))
+    }
+
     fn expanded_projection(
         &self,
         projection: Option<&Vec>,
@@ -525,10 +631,10 @@ impl IcebergTableProvider {
             let mut cols: Vec = used.clone();
             if let Some(expr) = conjunction(filters.iter().cloned()) {
                 for c in expr.column_refs() {
-                    if let Ok(idx) = self.arrow_schema.index_of(c.name.as_str()) {
-                        if !cols.contains(&idx) {
-                            cols.push(idx);
-                        }
+                    if let Ok(idx) = self.arrow_schema.index_of(c.name.as_str())
+                        && !cols.contains(&idx)
+                    {
+                        cols.push(idx);
                     }
                 }
             }
@@ -682,7 +788,7 @@ impl IcebergTableProvider {
 #[async_trait]
 impl TableProvider for IcebergTableProvider {
     fn schema(&self) -> Arc {
-        self.arrow_schema.clone()
+        self.output_schema.clone()
     }
 
     fn table_type(&self) -> TableType {
@@ -712,6 +818,12 @@ impl TableProvider for IcebergTableProvider {
             )));
         };
 
+        if self.file_column_name.is_some() || self.row_index_column_name.is_some() {
+            return self
+                .scan_with_merge_metadata(session, projection, filters, limit)
+                .await;
+        }
+
         if self.metadata_as_data_read {
             return self
                 .scan_metadata_as_data(session, projection, filters, limit)
@@ -869,6 +981,9 @@ impl TableProvider for IcebergTableProvider {
             let file_scan_config =
                 FileScanConfigBuilder::new(object_store_url.clone(), parquet_source)
                     .with_file_groups(vec![FileGroup::from(partitioned)])
+                    // Position deletes require the original file order and absolute offsets.
+                    .with_partitioned_by_file_group(true)
+                    .with_preserve_order(true)
                     .with_expr_adapter(Some(Arc::new(SchemaEvolutionPhysicalExprAdapterFactory {})
                         as Arc))
                     .build();
@@ -944,7 +1059,10 @@ impl TableProvider for IcebergTableProvider {
         &self,
         filter: &[&Expr],
     ) -> Result> {
-        if self.metadata_as_data_read {
+        if self.metadata_as_data_read
+            || self.file_column_name.is_some()
+            || self.row_index_column_name.is_some()
+        {
             return Ok(vec![TableProviderFilterPushDown::Unsupported; filter.len()]);
         }
         Ok(filter
@@ -955,6 +1073,205 @@ impl TableProvider for IcebergTableProvider {
 }
 
 impl IcebergTableProvider {
+    async fn scan_with_merge_metadata(
+        &self,
+        session: &dyn Session,
+        projection: Option<&Vec>,
+        filters: &[Expr],
+        limit: Option,
+    ) -> Result> {
+        log::trace!("Starting merge-metadata scan for table: {}", self.table_uri);
+
+        let Some(snapshot) = self.snapshot.as_ref() else {
+            return Ok(Arc::new(EmptyExec::new(
+                self.projected_arrow_schema(projection)?,
+            )));
+        };
+
+        let table_url = Url::parse(&self.table_uri)
+            .map_err(|e| datafusion::common::DataFusionError::External(Box::new(e)))?;
+        let base_store = get_object_store_from_session(session, &table_url)?;
+        let store_ctx = StoreContext::new(base_store.clone(), &table_url)?;
+
+        let manifest_list = self.load_manifest_list(&store_ctx).await?;
+        let (pruning_filters, parquet_pushdown_filters) = self.separate_filters(filters);
+        let mut data_files_with_seq = self
+            .load_data_files_with_seq(session, &pruning_filters, &store_ctx, &manifest_list)
+            .await?;
+
+        let filter_expr = conjunction(pruning_filters.iter().cloned());
+        if filter_expr.is_some() || limit.is_some() {
+            let (files_only, seqs_only): (Vec, Vec) =
+                data_files_with_seq.iter().cloned().unzip();
+            let seq_by_path: HashMap = files_only
+                .iter()
+                .map(|f| f.file_path.clone())
+                .zip(seqs_only)
+                .collect();
+            let (kept, _mask) = prune_files(
+                session,
+                &pruning_filters,
+                limit,
+                self.rebuild_logical_schema_for_filters(None, filters),
+                files_only,
+                &self.schema,
+            )?;
+            data_files_with_seq = kept
+                .into_iter()
+                .map(|df| {
+                    let seq = seq_by_path.get(&df.file_path).copied().unwrap_or(0);
+                    (df, seq)
+                })
+                .collect();
+        }
+
+        if data_files_with_seq.is_empty() {
+            return Ok(Arc::new(EmptyExec::new(
+                self.projected_arrow_schema(projection)?,
+            )));
+        }
+
+        let delete_index = self
+            .build_delete_file_index(&store_ctx, &manifest_list)
+            .await?;
+        let object_store_url = self.object_store_url()?;
+        let file_column_name = self.file_column_name.as_ref().ok_or_else(|| {
+            datafusion::common::DataFusionError::Internal(
+                "Iceberg merge metadata scan requires a file column".to_string(),
+            )
+        })?;
+        let mut clean_files = Vec::new();
+        let mut dirty_units = Vec::new();
+
+        for (data_file, sequence_number) in data_files_with_seq {
+            let matched = delete_index.for_data_file(&data_file, sequence_number);
+            if matched.is_empty() {
+                clean_files.push(data_file);
+            } else {
+                dirty_units.push((data_file, matched.positional, matched.equality));
+            }
+        }
+
+        let mut branches: Vec> =
+            Vec::with_capacity(dirty_units.len() + usize::from(!clean_files.is_empty()));
+
+        if !clean_files.is_empty() {
+            let partitioned_files = self.create_merge_partitioned_files(&store_ctx, clean_files)?;
+            let file_groups = partitioned_files
+                .into_iter()
+                .map(|file| FileGroup::from(vec![file]))
+                .collect::>();
+            let parquet_source =
+                self.build_merge_parquet_source(session, file_column_name.as_str());
+            let file_scan_config =
+                FileScanConfigBuilder::new(object_store_url.clone(), parquet_source)
+                    .with_file_groups(file_groups)
+                    // MERGE synthesizes file-local row positions before applying filters.
+                    // Keep every file group whole so DataFusion cannot split a Parquet file
+                    // into byte ranges whose streams would each start at position zero.
+                    .with_partitioned_by_file_group(true)
+                    .with_preserve_order(true)
+                    .with_expr_adapter(Some(Arc::new(SchemaEvolutionPhysicalExprAdapterFactory {})
+                        as Arc))
+                    .build();
+            let data_scan = DataSourceExec::from_data_source(file_scan_config);
+            branches.push(Arc::new(
+                IcebergMergeMetadataExec::try_new_partitioned_files(
+                    data_scan,
+                    file_column_name.clone(),
+                    self.row_index_column_name.clone(),
+                )?,
+            ));
+        }
+
+        for (df, positional_deletes, equality_deletes) in dirty_units {
+            let partitioned = self.create_partitioned_files(&store_ctx, vec![df.clone()])?;
+            let parquet_source = self.build_parquet_source(session, None, &[], &[], false)?;
+            let file_scan_config =
+                FileScanConfigBuilder::new(object_store_url.clone(), parquet_source)
+                    .with_file_groups(vec![FileGroup::from(partitioned)])
+                    // Existing position deletes and MERGE row positions both require the
+                    // original file order and absolute offsets.
+                    .with_partitioned_by_file_group(true)
+                    .with_preserve_order(true)
+                    .with_expr_adapter(Some(Arc::new(SchemaEvolutionPhysicalExprAdapterFactory {})
+                        as Arc))
+                    .build();
+            let data_scan: Arc =
+                DataSourceExec::from_data_source(file_scan_config);
+            let with_metadata: Arc =
+                Arc::new(IcebergMergeMetadataExec::try_new(
+                    data_scan,
+                    df.file_path.clone(),
+                    df.partition_spec_id,
+                    serde_json::to_string(&df.partition).map_err(|error| {
+                        datafusion::common::DataFusionError::External(Box::new(error))
+                    })?,
+                    self.file_column_name.clone(),
+                    self.row_index_column_name.clone(),
+                )?);
+
+            branches.push(Arc::new(IcebergDeleteApplyExec::new(
+                with_metadata,
+                df.file_path.clone(),
+                positional_deletes,
+                equality_deletes,
+                self.table_uri.clone(),
+                self.schema.clone(),
+            )));
+        }
+
+        let unioned: Arc = if branches.len() == 1 {
+            branches.into_iter().next().ok_or_else(|| {
+                datafusion::common::DataFusionError::Internal(
+                    "unreachable: branches.len() == 1 but next() returned None".to_string(),
+                )
+            })?
+        } else {
+            UnionExec::try_new(branches)?
+        };
+
+        let after_filter: Arc = if !parquet_pushdown_filters.is_empty() {
+            let df_schema = self.output_schema.clone().to_dfschema()?;
+            let pushdown_expr = conjunction(parquet_pushdown_filters.clone()).ok_or_else(|| {
+                datafusion::common::DataFusionError::Internal(
+                    "conjunction over non-empty filters returned None".to_string(),
+                )
+            })?;
+            let simplified = simplify_expr(session, &df_schema, pushdown_expr)?;
+            Arc::new(FilterExec::try_new(simplified, unioned)?)
+        } else {
+            unioned
+        };
+
+        let after_projection: Arc = if let Some(proj) = projection {
+            let projected_schema = self.output_schema.clone();
+            let proj_exprs: Vec<(Arc, String)> = proj
+                .iter()
+                .map(|&idx| {
+                    let field = projected_schema.field(idx);
+                    let col: Arc = Arc::new(Column::new(field.name(), idx));
+                    (col, field.name().to_string())
+                })
+                .collect();
+            Arc::new(ProjectionExec::try_new(proj_exprs, after_filter)?)
+        } else {
+            after_filter
+        };
+
+        let final_plan: Arc = if let Some(lim) = limit {
+            Arc::new(GlobalLimitExec::new(after_projection, 0, Some(lim)))
+        } else {
+            after_projection
+        };
+
+        log::trace!(
+            "Built merge-metadata scan for snapshot {}",
+            snapshot.snapshot_id()
+        );
+        Ok(final_plan)
+    }
+
     fn classify_pushdown_for_expr(&self, expr: &Expr) -> TableProviderFilterPushDown {
         use TableProviderFilterPushDown as FP;
         // Identity partition columns can satisfy Eq/IN at partition level. Transformed
@@ -967,30 +1284,30 @@ impl IcebergTableProvider {
                     Operator::Eq => {
                         if let (Some(col), true) =
                             (self.expr_as_column_name(l), self.expr_is_literal(r))
+                            && let Some(pushdown) = self.eq_in_pushdown_for_partition_col(&col)
                         {
-                            if let Some(pushdown) = self.eq_in_pushdown_for_partition_col(&col) {
-                                return pushdown;
-                            }
+                            return pushdown;
                         }
                         if let (Some(col), true) =
                             (self.expr_as_column_name(r), self.expr_is_literal(l))
+                            && let Some(pushdown) = self.eq_in_pushdown_for_partition_col(&col)
                         {
-                            if let Some(pushdown) = self.eq_in_pushdown_for_partition_col(&col) {
-                                return pushdown;
-                            }
+                            return pushdown;
                         }
                         FP::Unsupported
                     }
                     Operator::Gt | Operator::GtEq | Operator::Lt | Operator::LtEq => {
-                        if let Some(col) = self.expr_as_column_name(l) {
-                            if self.expr_is_literal(r) && self.is_partition_source_col(&col) {
-                                return FP::Inexact;
-                            }
+                        if let Some(col) = self.expr_as_column_name(l)
+                            && self.expr_is_literal(r)
+                            && self.is_partition_source_col(&col)
+                        {
+                            return FP::Inexact;
                         }
-                        if let Some(col) = self.expr_as_column_name(r) {
-                            if self.expr_is_literal(l) && self.is_partition_source_col(&col) {
-                                return FP::Inexact;
-                            }
+                        if let Some(col) = self.expr_as_column_name(r)
+                            && self.expr_is_literal(l)
+                            && self.is_partition_source_col(&col)
+                        {
+                            return FP::Inexact;
                         }
                         FP::Unsupported
                     }
@@ -1090,11 +1407,11 @@ impl IcebergTableProvider {
                 if matches!(pf.transform, Transform::Void | Transform::Unknown) {
                     continue;
                 }
-                if let Some(field) = self.schema.field_by_id(pf.source_id) {
-                    if field.name == col_name {
-                        found = true;
-                        only_identity &= matches!(pf.transform, Transform::Identity);
-                    }
+                if let Some(field) = self.schema.field_by_id(pf.source_id)
+                    && field.name == col_name
+                {
+                    found = true;
+                    only_identity &= matches!(pf.transform, Transform::Identity);
                 }
             }
         }
@@ -1113,12 +1430,11 @@ impl IcebergTableProvider {
             }
             if let Some(expr) = conjunction(filters.iter().cloned()) {
                 for c in expr.column_refs() {
-                    if let Ok(idx) = self.arrow_schema.index_of(c.name.as_str()) {
-                        if !used.contains(&idx)
-                            && !fields.iter().any(|f| f.name() == c.name.as_str())
-                        {
-                            fields.push(Arc::new(self.arrow_schema.field(idx).clone()));
-                        }
+                    if let Ok(idx) = self.arrow_schema.index_of(c.name.as_str())
+                        && !used.contains(&idx)
+                        && !fields.iter().any(|f| f.name() == c.name.as_str())
+                    {
+                        fields.push(Arc::new(self.arrow_schema.field(idx).clone()));
                     }
                 }
             }
diff --git a/crates/sail-iceberg/src/datasource/pruning.rs b/crates/sail-iceberg/src/datasource/pruning.rs
index 490cc0d37..a0b54c0ed 100644
--- a/crates/sail-iceberg/src/datasource/pruning.rs
+++ b/crates/sail-iceberg/src/datasource/pruning.rs
@@ -361,27 +361,27 @@ fn collect_source_eq_filters(schema: &Schema, filters: &[Expr]) -> Vec<(i32, Pri
                 let r = strip(right);
 
                 // col = lit
-                if let Expr::Column(c) = l {
-                    if let Expr::Literal(sv, _) = r {
-                        let col_name = c.name.clone();
-                        if let Some(field) = schema.field_by_name(&col_name) {
-                            if let Ok(pl) = scalar_to_primitive_literal(sv) {
-                                acc.push((field.id, pl));
-                                return;
-                            }
-                        }
+                if let Expr::Column(c) = l
+                    && let Expr::Literal(sv, _) = r
+                {
+                    let col_name = c.name.clone();
+                    if let Some(field) = schema.field_by_name(&col_name)
+                        && let Ok(pl) = scalar_to_primitive_literal(sv)
+                    {
+                        acc.push((field.id, pl));
+                        return;
                     }
                 }
 
                 // lit = col
-                if let Expr::Literal(sv, _) = l {
-                    if let Expr::Column(c) = r {
-                        let col_name = c.name.clone();
-                        if let Some(field) = schema.field_by_name(&col_name) {
-                            if let Ok(pl) = scalar_to_primitive_literal(sv) {
-                                acc.push((field.id, pl));
-                            }
-                        }
+                if let Expr::Literal(sv, _) = l
+                    && let Expr::Column(c) = r
+                {
+                    let col_name = c.name.clone();
+                    if let Some(field) = schema.field_by_name(&col_name)
+                        && let Ok(pl) = scalar_to_primitive_literal(sv)
+                    {
+                        acc.push((field.id, pl));
                     }
                 }
             }
@@ -417,20 +417,20 @@ fn collect_source_in_filters(
         match e {
             Expr::InList(in_list) if !in_list.negated => {
                 let e = strip(&in_list.expr);
-                if let Expr::Column(c) = e {
-                    if let Some(field) = schema.field_by_name(&c.name) {
-                        let mut vals = Vec::new();
-                        for item in &in_list.list {
-                            if let Expr::Literal(ref sv, _) = item {
-                                if let Ok(pl) = scalar_to_primitive_literal(sv) {
-                                    vals.push(pl);
-                                }
-                            }
-                        }
-                        if !vals.is_empty() {
-                            acc.entry(field.id).or_default().extend(vals);
+                if let Expr::Column(c) = e
+                    && let Some(field) = schema.field_by_name(&c.name)
+                {
+                    let mut vals = Vec::new();
+                    for item in &in_list.list {
+                        if let Expr::Literal(sv, _) = item
+                            && let Ok(pl) = scalar_to_primitive_literal(sv)
+                        {
+                            vals.push(pl);
                         }
                     }
+                    if !vals.is_empty() {
+                        acc.entry(field.id).or_default().extend(vals);
+                    }
                 }
             }
             Expr::BinaryExpr(BinaryExpr { left, op, right }) if *op == Operator::And => {
@@ -510,10 +510,9 @@ fn partition_predicates_for_spec(
                     partition_field.transform,
                     source_type,
                     literal.clone(),
-                ) {
-                    if seen.insert(value.clone()) {
-                        values.push(value);
-                    }
+                ) && seen.insert(value.clone())
+                {
+                    values.push(value);
                 }
             }
             if !values.is_empty() {
@@ -524,15 +523,14 @@ fn partition_predicates_for_spec(
             }
         }
 
-        if let Some(range) = range_filters.get(&partition_field.source_id) {
-            if let Some(range) =
+        if let Some(range) = range_filters.get(&partition_field.source_id)
+            && let Some(range) =
                 transform_range_constraint(partition_field.transform, source_type, range)
-            {
-                predicates.push(PartitionPredicate {
-                    field_index,
-                    constraint: PartitionConstraint::Range(range),
-                });
-            }
+        {
+            predicates.push(PartitionPredicate {
+                field_index,
+                constraint: PartitionConstraint::Range(range),
+            });
         }
     }
 
@@ -663,15 +661,15 @@ fn literal_may_match_bounds(
     lower: Option<&PrimitiveLiteral>,
     upper: Option<&PrimitiveLiteral>,
 ) -> bool {
-    if let Some(lower) = lower {
-        if value < lower {
-            return false;
-        }
+    if let Some(lower) = lower
+        && value < lower
+    {
+        return false;
     }
-    if let Some(upper) = upper {
-        if value > upper {
-            return false;
-        }
+    if let Some(upper) = upper
+        && value > upper
+    {
+        return false;
     }
     true
 }
@@ -681,29 +679,29 @@ fn range_may_match_bounds(
     lower: Option<&PrimitiveLiteral>,
     upper: Option<&PrimitiveLiteral>,
 ) -> bool {
-    if let (Some((min, inclusive)), Some(upper)) = (&range.min, upper) {
-        if min > upper || (min == upper && !inclusive) {
-            return false;
-        }
+    if let (Some((min, inclusive)), Some(upper)) = (&range.min, upper)
+        && (min > upper || (min == upper && !inclusive))
+    {
+        return false;
     }
-    if let (Some((max, inclusive)), Some(lower)) = (&range.max, lower) {
-        if max < lower || (max == lower && !inclusive) {
-            return false;
-        }
+    if let (Some((max, inclusive)), Some(lower)) = (&range.max, lower)
+        && (max < lower || (max == lower && !inclusive))
+    {
+        return false;
     }
     true
 }
 
 fn literal_may_match_range(value: &PrimitiveLiteral, range: &RangeConstraint) -> bool {
-    if let Some((min, inclusive)) = &range.min {
-        if value < min || (value == min && !inclusive) {
-            return false;
-        }
+    if let Some((min, inclusive)) = &range.min
+        && (value < min || (value == min && !inclusive))
+    {
+        return false;
     }
-    if let Some((max, inclusive)) = &range.max {
-        if value > max || (value == max && !inclusive) {
-            return false;
-        }
+    if let Some((max, inclusive)) = &range.max
+        && (value > max || (value == max && !inclusive))
+    {
+        return false;
     }
     true
 }
@@ -723,7 +721,7 @@ fn collect_source_range_filters(
     fn tighten_min(cur: &mut Option<(PrimitiveLiteral, bool)>, cand: (PrimitiveLiteral, bool)) {
         match cur {
             None => *cur = Some(cand),
-            Some((ref mut v, ref mut incl)) => {
+            Some((v, incl)) => {
                 if cand.0 > *v || (cand.0 == *v && !cand.1 && *incl) {
                     *v = cand.0;
                     *incl = cand.1;
@@ -734,7 +732,7 @@ fn collect_source_range_filters(
     fn tighten_max(cur: &mut Option<(PrimitiveLiteral, bool)>, cand: (PrimitiveLiteral, bool)) {
         match cur {
             None => *cur = Some(cand),
-            Some((ref mut v, ref mut incl)) => {
+            Some((v, incl)) => {
                 if cand.0 < *v || (cand.0 == *v && !cand.1 && *incl) {
                     *v = cand.0;
                     *incl = cand.1;
@@ -752,11 +750,11 @@ fn collect_source_range_filters(
         literal: &datafusion::common::scalar::ScalarValue,
         inclusive: bool,
     ) {
-        if let Some(field) = schema.field_by_name(column_name) {
-            if let Ok(pl) = scalar_to_primitive_literal(literal) {
-                let entry = acc.entry(field.id).or_default();
-                tighten_min(&mut entry.min, (pl, inclusive));
-            }
+        if let Some(field) = schema.field_by_name(column_name)
+            && let Ok(pl) = scalar_to_primitive_literal(literal)
+        {
+            let entry = acc.entry(field.id).or_default();
+            tighten_min(&mut entry.min, (pl, inclusive));
         }
     }
 
@@ -767,11 +765,11 @@ fn collect_source_range_filters(
         literal: &datafusion::common::scalar::ScalarValue,
         inclusive: bool,
     ) {
-        if let Some(field) = schema.field_by_name(column_name) {
-            if let Ok(pl) = scalar_to_primitive_literal(literal) {
-                let entry = acc.entry(field.id).or_default();
-                tighten_max(&mut entry.max, (pl, inclusive));
-            }
+        if let Some(field) = schema.field_by_name(column_name)
+            && let Ok(pl) = scalar_to_primitive_literal(literal)
+        {
+            let entry = acc.entry(field.id).or_default();
+            tighten_max(&mut entry.max, (pl, inclusive));
         }
     }
 
diff --git a/crates/sail-iceberg/src/datasource/type_converter.rs b/crates/sail-iceberg/src/datasource/type_converter.rs
index c18f273f2..4ef05aee3 100644
--- a/crates/sail-iceberg/src/datasource/type_converter.rs
+++ b/crates/sail-iceberg/src/datasource/type_converter.rs
@@ -15,10 +15,10 @@ use std::sync::Arc;
 
 use arrow_schema::extension::ExtensionType;
 use datafusion::arrow::datatypes::{
-    validate_decimal_precision_and_scale, DataType as ArrowDataType,
-    Decimal128Type as ArrowDecimal128Type, Field as ArrowField, Schema as ArrowSchema, TimeUnit,
+    DataType as ArrowDataType, Decimal128Type as ArrowDecimal128Type, Field as ArrowField,
+    Schema as ArrowSchema, TimeUnit, validate_decimal_precision_and_scale,
 };
-use datafusion_common::{plan_datafusion_err, plan_err, Result};
+use datafusion_common::{Result, plan_datafusion_err, plan_err};
 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
 use parquet_variant_compute::VariantType;
 use rust_decimal::prelude::ToPrimitive;
@@ -29,9 +29,9 @@ use sail_common_datafusion::variant::{
 };
 use serde_json;
 
+use crate::ICEBERG_LIST_FIELD_NAME;
 use crate::spec::types::values::Literal;
 use crate::spec::{ListType, MapType, NestedField, PrimitiveType, Schema, StructType, Type};
-use crate::ICEBERG_LIST_FIELD_NAME;
 
 pub const ICEBERG_ARROW_FIELD_DOC_KEY: &str = "doc";
 pub const ICEBERG_FIELD_INITIAL_DEFAULT: &str = "iceberg.field.initial-default";
diff --git a/crates/sail-iceberg/src/io/mod.rs b/crates/sail-iceberg/src/io/mod.rs
index 054cf1b75..a84ce3618 100644
--- a/crates/sail-iceberg/src/io/mod.rs
+++ b/crates/sail-iceberg/src/io/mod.rs
@@ -13,8 +13,8 @@
 use std::sync::Arc;
 
 use datafusion::common::DataFusionError;
-use object_store::path::Path as ObjectPath;
 use object_store::ObjectStoreExt;
+use object_store::path::Path as ObjectPath;
 use url::Url;
 
 use crate::spec::{FormatVersion, Manifest, ManifestList};
diff --git a/crates/sail-iceberg/src/lib.rs b/crates/sail-iceberg/src/lib.rs
index b7cb29b69..da963916c 100644
--- a/crates/sail-iceberg/src/lib.rs
+++ b/crates/sail-iceberg/src/lib.rs
@@ -20,6 +20,7 @@ pub mod options;
 pub mod physical;
 pub mod physical_plan;
 pub(crate) mod properties;
+pub(crate) mod row_level_metadata;
 pub mod schema_evolution;
 pub mod spec;
 pub mod table;
@@ -29,18 +30,18 @@ pub mod utils;
 pub use datasource::type_converter::*;
 pub use datasource::*;
 pub use logical::IcebergTableSource;
+pub use operations::Transaction;
 pub use operations::action::*;
 pub use operations::append::*;
 pub use operations::snapshot::*;
 pub use operations::write::base_writer::*;
 pub use operations::write::file_writer::*;
 pub use operations::write::{IcebergWriter, WriteOutcome};
-pub use operations::Transaction;
 pub use physical::IcebergPhysicalPlanner;
+pub use physical_plan::IcebergWriterExecOptions;
 pub use physical_plan::discovery_exec::IcebergDiscoveryExec;
-pub use physical_plan::manifest_scan_exec::{manifest_scan_schema, IcebergManifestScanExec};
+pub use physical_plan::manifest_scan_exec::{IcebergManifestScanExec, manifest_scan_schema};
 pub use physical_plan::scan_by_data_files_exec::IcebergScanByDataFilesExec;
-pub use physical_plan::IcebergWriterExecOptions;
 pub use schema_evolution::*;
 pub use spec::*;
 pub use table_format::*;
diff --git a/crates/sail-iceberg/src/logical/merge.rs b/crates/sail-iceberg/src/logical/merge.rs
new file mode 100644
index 000000000..fce54e128
--- /dev/null
+++ b/crates/sail-iceberg/src/logical/merge.rs
@@ -0,0 +1,277 @@
+use std::sync::Arc;
+
+use datafusion::logical_expr::logical_plan::builder::LogicalPlanBuilder;
+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
+use datafusion_common::{Column, Result, not_impl_err};
+use datafusion_expr::logical_plan::Extension;
+use datafusion_expr::{Expr, LogicalPlan, TableScan, TableSource};
+use log::trace;
+use sail_common_datafusion::datasource::{
+    MERGE_FILE_COLUMN, MERGE_ROW_INDEX_COLUMN, MergeCapableSource, MergeInfo, MergeMatchedAction,
+    MergeNotMatchedBySourceAction,
+};
+use sail_logical_plan::merge::{RowLevelWriteNode, expand_merge, validate_merge_internal_columns};
+
+use crate::logical::table_source::IcebergTableSource;
+use crate::row_level_metadata::{MERGE_PARTITION_COLUMN, MERGE_PARTITION_SPEC_ID_COLUMN};
+
+/// Expand MERGE information into a unified row-level write node for Iceberg.
+///
+/// Iceberg MERGE is planned as merge-on-read: target rows affected by DELETE or
+/// UPDATE clauses are represented by position deletes, and UPDATE/INSERT output
+/// rows are appended as new data files.
+pub fn expand_merge_node(info: MergeInfo) -> Result {
+    // TODO: Add Iceberg MERGE schema evolution support.
+    if info.options.with_schema_evolution {
+        return not_impl_err!("Iceberg MERGE WITH SCHEMA EVOLUTION is not supported");
+    }
+    validate_merge_internal_columns(
+        &info,
+        &[
+            MERGE_FILE_COLUMN,
+            MERGE_ROW_INDEX_COLUMN,
+            MERGE_PARTITION_SPEC_ID_COLUMN,
+            MERGE_PARTITION_COLUMN,
+        ],
+    )?;
+    let expected_snapshot_id = Some(merge_target_snapshot_id(info.target.as_ref())?);
+    let row_index_column = merge_needs_position_deletes(&info).then_some(MERGE_ROW_INDEX_COLUMN);
+    let mut target_plan = ensure_merge_metadata_columns(
+        info.target.as_ref().clone(),
+        MERGE_FILE_COLUMN,
+        row_index_column,
+    )?;
+    let target_fields: Vec = target_plan
+        .schema()
+        .fields()
+        .iter()
+        .map(|f| f.name().clone())
+        .collect();
+    trace!(
+        "iceberg merge target schema after metadata columns: {:?}",
+        target_fields
+    );
+    let mut required_metadata_columns = vec![
+        MERGE_FILE_COLUMN,
+        MERGE_PARTITION_SPEC_ID_COLUMN,
+        MERGE_PARTITION_COLUMN,
+    ];
+    if let Some(row_index_column) = row_index_column {
+        required_metadata_columns.push(row_index_column);
+    }
+    if required_metadata_columns
+        .iter()
+        .any(|column| !target_fields.iter().any(|name| name == column))
+    {
+        let mut exprs: Vec = target_fields
+            .iter()
+            .map(|name| Expr::Column(Column::from_name(name.clone())))
+            .collect();
+        for metadata_column in required_metadata_columns {
+            if !target_fields.iter().any(|name| name == metadata_column) {
+                exprs.push(Expr::Column(Column::from_name(metadata_column)).alias(metadata_column));
+            }
+        }
+        target_plan = LogicalPlanBuilder::from(target_plan)
+            .project(exprs)?
+            .build()?;
+    }
+
+    let info = MergeInfo {
+        target: Arc::new(target_plan),
+        source: info.source,
+        options: info.options,
+        input_schema: info.input_schema,
+    };
+    let raw_target = Arc::clone(&info.target);
+    let raw_source = Arc::clone(&info.source);
+    let raw_input_schema = info.input_schema.clone();
+    let expansion = expand_merge(
+        info,
+        MERGE_FILE_COLUMN,
+        row_index_column,
+        &[MERGE_PARTITION_SPEC_ID_COLUMN, MERGE_PARTITION_COLUMN],
+    )?;
+    // Iceberg consumes delete metadata from the write plan. RowLevelWriteNode still
+    // requires a touched-file input, so use an empty placeholder instead of planning
+    // cloned joins that would duplicate the source and target scans.
+    let placeholder_touched_plan = LogicalPlanBuilder::empty(false).build()?;
+    let write_node = RowLevelWriteNode::new_merge(
+        raw_target,
+        raw_source,
+        raw_input_schema,
+        Arc::new(expansion.write_plan),
+        Arc::new(placeholder_touched_plan),
+        None,
+        expansion.options,
+        expansion.output_schema,
+    )
+    .with_expected_snapshot_id(expected_snapshot_id);
+
+    Ok(LogicalPlan::Extension(Extension {
+        node: Arc::new(write_node),
+    }))
+}
+
+fn merge_target_snapshot_id(plan: &LogicalPlan) -> Result> {
+    let mut snapshot_id = None;
+    plan.apply(|node| {
+        if let LogicalPlan::TableScan(scan) = node
+            && let Some(source) = scan.source.downcast_ref::()
+        {
+            snapshot_id = source
+                .provider()
+                .current_snapshot()
+                .map(|snapshot| snapshot.snapshot_id());
+            return Ok(TreeNodeRecursion::Stop);
+        }
+        Ok(TreeNodeRecursion::Continue)
+    })?;
+    Ok(snapshot_id)
+}
+
+fn merge_needs_position_deletes(info: &MergeInfo) -> bool {
+    info.options.matched_clauses.iter().any(|clause| {
+        matches!(
+            clause.action,
+            MergeMatchedAction::Delete
+                | MergeMatchedAction::UpdateAll
+                | MergeMatchedAction::UpdateSet(_)
+        )
+    }) || info
+        .options
+        .not_matched_by_source_clauses
+        .iter()
+        .any(|clause| {
+            matches!(
+                clause.action,
+                MergeNotMatchedBySourceAction::Delete | MergeNotMatchedBySourceAction::UpdateSet(_)
+            )
+        })
+}
+
+fn try_enable_merge_metadata_columns(
+    source: &Arc,
+    file_col: &str,
+    row_index_col: Option<&str>,
+) -> Result<
+    Option<(
+        Arc,
+        datafusion::arrow::datatypes::SchemaRef,
+    )>,
+> {
+    let Some(iceberg_source) = source.downcast_ref::() else {
+        return Ok(None);
+    };
+    let mut new_source = Arc::clone(source);
+    let mut changed = false;
+
+    if iceberg_source.file_column_name().is_none() {
+        new_source = iceberg_source.with_file_column(file_col)?;
+        changed = true;
+    }
+    if let (Some(row_index_col), Some(iceberg_source)) = (
+        row_index_col,
+        new_source.downcast_ref::(),
+    ) && iceberg_source.row_index_column_name().is_none()
+    {
+        new_source = iceberg_source.with_row_index_column(row_index_col)?;
+        changed = true;
+    }
+    if changed {
+        let schema = new_source.schema();
+        return Ok(Some((new_source, schema)));
+    }
+    Ok(None)
+}
+
+fn ensure_merge_metadata_columns(
+    plan: LogicalPlan,
+    file_col: &str,
+    row_index_col: Option<&str>,
+) -> Result {
+    let mut metadata_cols = vec![
+        file_col,
+        MERGE_PARTITION_SPEC_ID_COLUMN,
+        MERGE_PARTITION_COLUMN,
+    ];
+    if let Some(row_index_col) = row_index_col {
+        metadata_cols.push(row_index_col);
+    }
+
+    let transformed = plan
+        .transform_up(|plan| {
+            if let LogicalPlan::TableScan(scan) = &plan
+                && let Some((new_source, schema)) =
+                    try_enable_merge_metadata_columns(&scan.source, file_col, row_index_col)?
+            {
+                let mut projection: Option> = scan.projection.clone();
+                if projection.is_none() {
+                    projection = Some((0..schema.fields().len()).collect::>());
+                }
+                if let Some(proj) = projection.as_mut() {
+                    for col in &metadata_cols {
+                        if let Some(idx) = schema.column_with_name(col).map(|(idx, _)| idx)
+                            && !proj.contains(&idx)
+                        {
+                            proj.push(idx);
+                        }
+                    }
+                }
+
+                let new_scan = LogicalPlan::TableScan(TableScan::try_new(
+                    scan.table_name.clone(),
+                    new_source,
+                    projection,
+                    scan.filters.clone(),
+                    scan.fetch,
+                )?);
+                return Ok(Transformed::yes(new_scan));
+            }
+
+            if let LogicalPlan::Projection(proj) = &plan {
+                let input_schema = proj.input.schema();
+                let mut new_exprs = proj.expr.clone();
+                let mut changed = false;
+                for col in &metadata_cols {
+                    let has_in_input = input_schema.fields().iter().any(|f| f.name() == *col);
+                    let has_in_projection = proj.expr.iter().any(|e| match e {
+                        Expr::Column(c) => c.name == *col,
+                        Expr::Alias(a) => a.name == *col,
+                        _ => false,
+                    });
+                    if has_in_input && !has_in_projection {
+                        new_exprs.push(Expr::Column(Column::from_name(*col)).alias(*col));
+                        changed = true;
+                    }
+                }
+                if changed {
+                    let new_proj = LogicalPlanBuilder::from(proj.input.as_ref().clone())
+                        .project(new_exprs)?
+                        .build()?;
+                    return Ok(Transformed::yes(new_proj));
+                }
+            }
+
+            Ok(Transformed::no(plan))
+        })
+        .map(|t| t.data)?;
+
+    let mut transformed = transformed;
+    if let LogicalPlan::SubqueryAlias(sa) = &transformed {
+        let missing_in_alias = metadata_cols.iter().any(|col| {
+            let has_in_child = sa.input.schema().fields().iter().any(|f| f.name() == *col);
+            let has_in_alias = sa.schema.fields().iter().any(|f| f.name() == *col);
+            has_in_child && !has_in_alias
+        });
+        if missing_in_alias {
+            transformed =
+                LogicalPlan::SubqueryAlias(datafusion_expr::logical_plan::SubqueryAlias::try_new(
+                    sa.input.clone(),
+                    sa.alias.clone(),
+                )?);
+        }
+    }
+
+    Ok(transformed)
+}
diff --git a/crates/sail-iceberg/src/logical/mod.rs b/crates/sail-iceberg/src/logical/mod.rs
index f00876652..c76c30117 100644
--- a/crates/sail-iceberg/src/logical/mod.rs
+++ b/crates/sail-iceberg/src/logical/mod.rs
@@ -1,3 +1,4 @@
+pub mod merge;
 pub mod table_source;
 
 pub use table_source::IcebergTableSource;
diff --git a/crates/sail-iceberg/src/logical/table_source.rs b/crates/sail-iceberg/src/logical/table_source.rs
index 62f717e17..9b7d9db24 100644
--- a/crates/sail-iceberg/src/logical/table_source.rs
+++ b/crates/sail-iceberg/src/logical/table_source.rs
@@ -4,6 +4,7 @@ use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::common::Result;
 use datafusion::datasource::TableProvider;
 use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource};
+use sail_common_datafusion::datasource::MergeCapableSource;
 
 use crate::datasource::provider::IcebergTableProvider;
 
@@ -40,6 +41,28 @@ impl IcebergTableSource {
     }
 }
 
+impl MergeCapableSource for IcebergTableSource {
+    fn file_column_name(&self) -> Option<&str> {
+        self.provider.file_column_name()
+    }
+
+    fn row_index_column_name(&self) -> Option<&str> {
+        self.provider.row_index_column_name()
+    }
+
+    fn with_file_column(&self, name: &str) -> Result> {
+        Ok(Arc::new(Self::new(Arc::new(
+            self.provider.as_ref().clone().with_file_column(name)?,
+        ))))
+    }
+
+    fn with_row_index_column(&self, name: &str) -> Result> {
+        Ok(Arc::new(Self::new(Arc::new(
+            self.provider.as_ref().clone().with_row_index_column(name)?,
+        ))))
+    }
+}
+
 impl TableSource for IcebergTableSource {
     fn schema(&self) -> SchemaRef {
         self.provider.schema()
diff --git a/crates/sail-iceberg/src/operations/append.rs b/crates/sail-iceberg/src/operations/append.rs
index 64cfd8595..51ded97d9 100644
--- a/crates/sail-iceberg/src/operations/append.rs
+++ b/crates/sail-iceberg/src/operations/append.rs
@@ -16,13 +16,11 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use uuid::Uuid;
 
-use super::{
-    ActionCommit, SnapshotProduceOperation, SnapshotProducer, Transaction, TransactionAction,
-};
+use super::{ActionCommit, SnapshotProducer, SnapshotUpdateKind, Transaction, TransactionAction};
 use crate::io::StoreContext;
 use crate::spec::manifest::ManifestMetadata;
 use crate::spec::manifest_list::ManifestList;
-use crate::spec::DataFile;
+use crate::spec::{DataFile, PartitionSpec};
 
 pub struct FastAppendAction {
     check_duplicate: bool,
@@ -33,6 +31,7 @@ pub struct FastAppendAction {
     parent_manifest_list: Option,
     store_ctx: Option,
     manifest_metadata: Option,
+    partition_specs: Vec,
     row_lineage_start_row_id: Option,
 }
 
@@ -53,6 +52,7 @@ impl FastAppendAction {
             parent_manifest_list: None,
             store_ctx: None,
             manifest_metadata: None,
+            partition_specs: Vec::new(),
             row_lineage_start_row_id: None,
         }
     }
@@ -96,6 +96,11 @@ impl FastAppendAction {
         self
     }
 
+    pub fn with_partition_specs(mut self, partition_specs: Vec) -> Self {
+        self.partition_specs = partition_specs;
+        self
+    }
+
     pub fn with_row_lineage_start_row_id(mut self, start_row_id: Option) -> Self {
         self.row_lineage_start_row_id = start_row_id;
         self
@@ -111,6 +116,7 @@ impl TransactionAction for FastAppendAction {
             self.store_ctx.clone(),
             self.manifest_metadata.clone(),
         )
+        .with_partition_specs(self.partition_specs.clone())
         .with_row_lineage_start_row_id(self.row_lineage_start_row_id);
         snapshot_producer.validate_added_data_files(&self.added_data_files)?;
 
@@ -118,13 +124,8 @@ impl TransactionAction for FastAppendAction {
             // TODO: validate duplicate files later
         }
 
-        struct FastAppendOperation;
-        impl SnapshotProduceOperation for FastAppendOperation {
-            fn operation(&self) -> &'static str {
-                "append"
-            }
-        }
-
-        snapshot_producer.commit(FastAppendOperation).await
+        snapshot_producer
+            .commit(SnapshotUpdateKind::FastAppend)
+            .await
     }
 }
diff --git a/crates/sail-iceberg/src/operations/bootstrap.rs b/crates/sail-iceberg/src/operations/bootstrap.rs
index 436df0731..42459527d 100644
--- a/crates/sail-iceberg/src/operations/bootstrap.rs
+++ b/crates/sail-iceberg/src/operations/bootstrap.rs
@@ -25,7 +25,7 @@ use url::Url;
 
 use crate::io::StoreContext;
 use crate::operations::helpers::format_version_for_schema;
-use crate::operations::{ActionCommit, SnapshotProduceOperation, SnapshotProducer, Transaction};
+use crate::operations::{ActionCommit, SnapshotProducer, Transaction};
 use crate::physical_plan::commit::IcebergCommitInfo;
 use crate::spec::metadata::table_metadata::SnapshotLog;
 use crate::spec::partition::PartitionSpec;
@@ -61,15 +61,6 @@ pub struct BootstrapResult {
     pub metadata_file: String,
 }
 
-/// Bootstrap operation for SnapshotProducer
-struct BootstrapOperation;
-
-impl SnapshotProduceOperation for BootstrapOperation {
-    fn operation(&self) -> &'static str {
-        "append"
-    }
-}
-
 pub(crate) async fn bootstrap_snapshot_action_commit(
     table_url: &Url,
     store_ctx: &StoreContext,
@@ -102,11 +93,15 @@ pub(crate) async fn bootstrap_snapshot_action_commit(
         .build()
         .map_err(DataFusionError::Execution)?;
 
-    let tx = Transaction::new(table_url.to_string(), empty_snapshot);
+    let tx = Transaction::new(
+        table_url.to_string(),
+        empty_snapshot,
+        table_meta.last_sequence_number,
+    );
     let manifest_meta = crate::spec::manifest::ManifestMetadata::new(
         Arc::new(schema_iceberg.clone()),
         schema_iceberg.schema_id(),
-        partition_spec,
+        partition_spec.clone(),
         format_version,
         crate::spec::ManifestContentType::Data,
     );
@@ -118,11 +113,13 @@ pub(crate) async fn bootstrap_snapshot_action_commit(
         Some(manifest_meta),
     )
     .with_bootstrap(true)
+    .with_added_delete_files(commit_info.delete_files.clone())
+    .with_partition_specs(table_meta.partition_specs.clone())
     .with_row_lineage_start_row_id(row_lineage_start_row_id)
     .with_write_path_mode(WritePathMode::Absolute);
 
     producer
-        .commit(BootstrapOperation)
+        .commit(commit_info.snapshot_update_kind)
         .await
         .map_err(DataFusionError::Execution)
 }
@@ -182,7 +179,7 @@ pub async fn bootstrap_new_table_with_style(
         .build()
         .map_err(DataFusionError::Execution)?;
 
-    let tx = Transaction::new(table_url.to_string(), empty_snapshot);
+    let tx = Transaction::new(table_url.to_string(), empty_snapshot, 0);
     let manifest_meta = crate::spec::manifest::ManifestMetadata::new(
         Arc::new(iceberg_schema.clone()),
         iceberg_schema.schema_id(),
@@ -200,11 +197,13 @@ pub async fn bootstrap_new_table_with_style(
         Some(manifest_meta),
     )
     .with_bootstrap(true)
+    .with_added_delete_files(commit_info.delete_files.clone())
+    .with_partition_specs(vec![partition_spec.clone()])
     .with_row_lineage_start_row_id(row_lineage_start_row_id)
     .with_write_path_mode(WritePathMode::Absolute);
 
     let action_commit = producer
-        .commit(BootstrapOperation)
+        .commit(commit_info.snapshot_update_kind)
         .await
         .map_err(DataFusionError::Execution)?;
 
@@ -224,14 +223,14 @@ pub async fn bootstrap_new_table_with_style(
         format_version,
         table_uuid: None,
         location: table_url.to_string(),
-        last_sequence_number: 1,
+        last_sequence_number: snapshot.sequence_number(),
         last_updated_ms: commit_timestamp_ms,
         last_column_id: iceberg_schema.highest_field_id(),
         schemas: vec![iceberg_schema.clone()],
         current_schema_id: iceberg_schema.schema_id(),
         partition_specs: vec![partition_spec.clone()],
         default_spec_id: partition_spec.spec_id(),
-        last_partition_id: partition_spec.highest_field_id().unwrap_or(0),
+        last_partition_id: partition_spec.last_assigned_field_id(),
         properties: table_properties,
         current_snapshot_id: Some(snapshot.snapshot_id()),
         next_row_id: snapshot.added_rows.and_then(|added_rows| {
@@ -329,7 +328,7 @@ pub async fn bootstrap_empty_table_metadata(
         current_schema_id: iceberg_schema.schema_id(),
         partition_specs: vec![partition_spec.clone()],
         default_spec_id: partition_spec.spec_id(),
-        last_partition_id: partition_spec.highest_field_id().unwrap_or(0),
+        last_partition_id: partition_spec.last_assigned_field_id(),
         properties: table_properties,
         current_snapshot_id: Some(-1),
         next_row_id: (format_version >= FormatVersion::V3).then_some(0),
@@ -404,21 +403,31 @@ pub async fn replace_empty_table_metadata(
 
     let mut metadata_log = previous_metadata.metadata_log.clone();
     metadata_log.push(crate::spec::metadata::table_metadata::MetadataLog {
-        timestamp_ms: commit_timestamp_ms,
+        timestamp_ms: previous_metadata.last_updated_ms,
         metadata_file: latest_meta_path.to_string(),
     });
 
+    let next_row_id = if format_version >= FormatVersion::V3 {
+        let mut row_lineage_metadata = previous_metadata.clone();
+        row_lineage_metadata.format_version = format_version;
+        row_lineage_metadata.row_lineage_start_row_id()
+    } else {
+        None
+    };
+
     let last_column_id = previous_metadata
         .last_column_id
         .max(iceberg_schema.highest_field_id());
     let last_partition_id = previous_metadata
         .last_partition_id
-        .max(partition_spec.highest_field_id().unwrap_or(0));
+        .max(partition_spec.last_assigned_field_id());
     let mut schemas = previous_metadata.schemas.clone();
     schemas.push(iceberg_schema.clone());
     let mut partition_specs = previous_metadata.partition_specs.clone();
     partition_specs.push(partition_spec.clone());
 
+    // FIXME: Preserve snapshot history, non-main refs, and historical sort/statistics
+    // metadata when replacing table metadata.
     let mut table_meta = TableMetadata {
         format_version,
         table_uuid: previous_metadata.table_uuid,
@@ -433,7 +442,7 @@ pub async fn replace_empty_table_metadata(
         last_partition_id,
         properties: table_properties,
         current_snapshot_id: Some(-1),
-        next_row_id: (format_version >= FormatVersion::V3).then_some(0),
+        next_row_id,
         encryption_keys: previous_metadata.encryption_keys.clone(),
         snapshots: vec![],
         snapshot_log: vec![],
@@ -528,6 +537,7 @@ pub async fn bootstrap_first_snapshot(
         .ok_or_else(|| DataFusionError::Plan("No snapshot in bootstrap commit".to_string()))?;
 
     // Update table metadata with the new snapshot
+    let previous_metadata_timestamp_ms = table_meta.last_updated_ms;
     let commit_timestamp_ms = crate::utils::timestamp::monotonic_timestamp_ms();
     table_meta.current_snapshot_id = Some(snapshot.snapshot_id());
     table_meta.snapshots.push(snapshot.clone());
@@ -538,12 +548,12 @@ pub async fn bootstrap_first_snapshot(
     table_meta
         .metadata_log
         .push(crate::spec::metadata::table_metadata::MetadataLog {
-            timestamp_ms: commit_timestamp_ms,
+            timestamp_ms: previous_metadata_timestamp_ms,
             metadata_file: previous_metadata_file
                 .unwrap_or(latest_meta_path)
                 .to_string(),
         });
-    table_meta.last_sequence_number = 1;
+    table_meta.last_sequence_number = snapshot.sequence_number();
     table_meta.last_updated_ms = commit_timestamp_ms;
     if let Some(added_rows) = snapshot.added_rows {
         table_meta.advance_next_row_id(added_rows);
diff --git a/crates/sail-iceberg/src/operations/mod.rs b/crates/sail-iceberg/src/operations/mod.rs
index 52219ce61..0437f26f1 100644
--- a/crates/sail-iceberg/src/operations/mod.rs
+++ b/crates/sail-iceberg/src/operations/mod.rs
@@ -29,14 +29,16 @@ use crate::spec::Snapshot;
 pub struct Transaction {
     table_uri: String,
     snapshot: Snapshot,
+    last_sequence_number: i64,
     actions: Vec>,
 }
 
 impl Transaction {
-    pub fn new(table_uri: String, snapshot: Snapshot) -> Self {
+    pub fn new(table_uri: String, snapshot: Snapshot, last_sequence_number: i64) -> Self {
         Self {
             table_uri,
             snapshot,
+            last_sequence_number,
             actions: Vec::new(),
         }
     }
@@ -49,6 +51,13 @@ impl Transaction {
         &self.snapshot
     }
 
+    pub fn next_sequence_number(&self) -> Result {
+        self.last_sequence_number.checked_add(1).ok_or_else(|| {
+            "cannot allocate Iceberg snapshot sequence number: table sequence is exhausted"
+                .to_string()
+        })
+    }
+
     pub async fn commit(self, _summary_op: &str) -> Result {
         Err("commit is not implemented yet".to_string())
     }
diff --git a/crates/sail-iceberg/src/operations/overwrite.rs b/crates/sail-iceberg/src/operations/overwrite.rs
index 3d1367c8b..2a5ae2009 100644
--- a/crates/sail-iceberg/src/operations/overwrite.rs
+++ b/crates/sail-iceberg/src/operations/overwrite.rs
@@ -14,9 +14,7 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 
-use super::{
-    ActionCommit, SnapshotProduceOperation, SnapshotProducer, Transaction, TransactionAction,
-};
+use super::{ActionCommit, SnapshotProducer, SnapshotUpdateKind, Transaction, TransactionAction};
 
 pub struct OverwriteAction;
 
@@ -32,19 +30,14 @@ impl OverwriteAction {
     }
 }
 
-struct OverwriteOperation;
-impl SnapshotProduceOperation for OverwriteOperation {
-    fn operation(&self) -> &'static str {
-        "overwrite"
-    }
-}
-
 #[async_trait]
 impl TransactionAction for OverwriteAction {
     async fn commit(self: Arc, tx: &Transaction) -> Result {
         // TODO: Implement full overwrite semantics (predicate/partition replaces, conflict checks,
         // delete manifests) instead of relying solely on SnapshotProducer for added data files.
         let snapshot_producer = SnapshotProducer::new(tx, vec![], None, None);
-        snapshot_producer.commit(OverwriteOperation).await
+        snapshot_producer
+            .commit(SnapshotUpdateKind::FullOverwrite)
+            .await
     }
 }
diff --git a/crates/sail-iceberg/src/operations/snapshot.rs b/crates/sail-iceberg/src/operations/snapshot.rs
index 05d6ff64a..1846c1e0d 100644
--- a/crates/sail-iceberg/src/operations/snapshot.rs
+++ b/crates/sail-iceberg/src/operations/snapshot.rs
@@ -10,29 +10,328 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::BTreeMap;
+
 use bytes::Bytes;
 use object_store::ObjectStoreExt;
+use serde::{Deserialize, Serialize};
 
 use super::{ActionCommit, Transaction};
 use crate::io::StoreContext;
 use crate::spec::manifest::ManifestWriterBuilder;
 use crate::spec::manifest_list::ManifestListWriter;
 use crate::spec::{
-    DataFile, FormatVersion, ManifestContentType, Operation, PartitionSpec, Schema,
-    SnapshotBuilder, SnapshotReference, SnapshotRetention, TableRequirement, TableUpdate,
-    MAIN_BRANCH,
+    DataFile, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestStatus, Operation,
+    PartitionSpec, Schema, SnapshotBuilder, SnapshotReference, SnapshotRetention, TableRequirement,
+    TableUpdate,
 };
 use crate::utils::join_table_uri;
 
-pub trait SnapshotProduceOperation: Send + Sync {
-    fn operation(&self) -> &'static str;
+fn active_file_count(manifest: &crate::spec::manifest_list::ManifestFile) -> Option {
+    Some(i64::from(manifest.added_files_count?) + i64::from(manifest.existing_files_count?))
+}
+
+fn active_row_count(manifest: &crate::spec::manifest_list::ManifestFile) -> Option {
+    Some(manifest.added_rows_count? + manifest.existing_rows_count?)
+}
+
+#[derive(Debug, Clone, Copy)]
+enum DeleteRowTotalsBase<'a> {
+    Empty,
+    Parent(&'a crate::spec::snapshots::Summary),
+}
+
+impl DeleteRowTotalsBase<'_> {
+    fn previous_total(self, key: &str) -> Option {
+        match self {
+            Self::Empty => Some(0),
+            Self::Parent(parent_summary) => {
+                parent_summary.additional_properties.get(key)?.parse().ok()
+            }
+        }
+    }
+}
+
+fn with_manifest_totals<'a>(
+    mut summary: crate::spec::snapshots::Summary,
+    delete_totals_base: DeleteRowTotalsBase<'_>,
+    manifests: impl IntoIterator,
+    added_position_deletes: u64,
+    added_equality_deletes: u64,
+) -> crate::spec::snapshots::Summary {
+    let mut total_data_files = Some(0);
+    let mut total_delete_files = Some(0);
+    let mut total_records = Some(0);
+    for manifest in manifests {
+        match manifest.content {
+            ManifestContentType::Data => {
+                total_data_files = total_data_files
+                    .zip(active_file_count(manifest))
+                    .map(|(total, active)| total + active);
+                total_records = total_records
+                    .zip(active_row_count(manifest))
+                    .map(|(total, active)| total + active);
+            }
+            ManifestContentType::Deletes => {
+                total_delete_files = total_delete_files
+                    .zip(active_file_count(manifest))
+                    .map(|(total, active)| total + active);
+            }
+        }
+    }
+
+    if let Some(total_data_files) = total_data_files {
+        summary = summary.with_property("total-data-files", total_data_files.to_string());
+    }
+    if let Some(total_delete_files) = total_delete_files {
+        summary = summary.with_property("total-delete-files", total_delete_files.to_string());
+    }
+    if let Some(total_records) = total_records {
+        summary = summary.with_property("total-records", total_records.to_string());
+    }
+
+    if let Some(previous_position_deletes) =
+        delete_totals_base.previous_total("total-position-deletes")
+    {
+        summary = summary.with_property(
+            "total-position-deletes",
+            (previous_position_deletes + added_position_deletes).to_string(),
+        );
+    }
+    if let Some(previous_equality_deletes) =
+        delete_totals_base.previous_total("total-equality-deletes")
+    {
+        summary = summary.with_property(
+            "total-equality-deletes",
+            (previous_equality_deletes + added_equality_deletes).to_string(),
+        );
+    }
+    summary
+}
+
+fn manifest_inputs(
+    manifest_metadata: &crate::spec::manifest::ManifestMetadata,
+    partition_specs: &[PartitionSpec],
+    files: &[DataFile],
+    content: ManifestContentType,
+) -> Result)>, String> {
+    let mut files_by_spec = BTreeMap::>::new();
+    for file in files {
+        files_by_spec
+            .entry(file.partition_spec_id)
+            .or_default()
+            .push(file.clone());
+    }
+
+    let manifest_kind = match content {
+        ManifestContentType::Data => "data",
+        ManifestContentType::Deletes => "delete",
+    };
+
+    files_by_spec
+        .into_iter()
+        .map(|(spec_id, files)| {
+            let partition_spec = partition_specs
+                .iter()
+                .chain(std::iter::once(&manifest_metadata.partition_spec))
+                .find(|spec| spec.spec_id() == spec_id)
+                .cloned()
+                .ok_or_else(|| {
+                    format!(
+                        "cannot write Iceberg {manifest_kind} manifest: partition spec {spec_id} is missing from table metadata"
+                    )
+                })?;
+            let mut metadata = manifest_metadata.clone();
+            metadata.partition_spec = partition_spec;
+            metadata.content = content.clone();
+            Ok((metadata, files))
+        })
+        .collect()
+}
+
+fn data_manifest_inputs(
+    manifest_metadata: &crate::spec::manifest::ManifestMetadata,
+    partition_specs: &[PartitionSpec],
+    data_files: &[DataFile],
+) -> Result)>, String> {
+    manifest_inputs(
+        manifest_metadata,
+        partition_specs,
+        data_files,
+        ManifestContentType::Data,
+    )
+}
+
+fn delete_manifest_inputs(
+    manifest_metadata: &crate::spec::manifest::ManifestMetadata,
+    partition_specs: &[PartitionSpec],
+    delete_files: &[DataFile],
+) -> Result)>, String> {
+    manifest_inputs(
+        manifest_metadata,
+        partition_specs,
+        delete_files,
+        ManifestContentType::Deletes,
+    )
+}
+
+fn validate_delete_files_for_format(
+    format_version: FormatVersion,
+    delete_files: &[DataFile],
+) -> Result<(), String> {
+    if delete_files
+        .iter()
+        .any(|file| file.content == crate::spec::DataContentType::Data)
+    {
+        return Err("Iceberg delete manifest input contains a data file".to_string());
+    }
+    if format_version == FormatVersion::V1 && !delete_files.is_empty() {
+        return Err("Iceberg v1 snapshots cannot add delete files".to_string());
+    }
+    if format_version == FormatVersion::V3
+        && delete_files
+            .iter()
+            .any(|file| file.content == crate::spec::DataContentType::PositionDeletes)
+    {
+        return Err(
+            "Iceberg v3 snapshots cannot add position delete files; v3 requires deletion vectors"
+                .to_string(),
+        );
+    }
+    Ok(())
+}
+
+fn manifest_counts_are_complete(manifest_file: &crate::spec::manifest_list::ManifestFile) -> bool {
+    manifest_file.added_files_count.is_some()
+        && manifest_file.existing_files_count.is_some()
+        && manifest_file.deleted_files_count.is_some()
+        && manifest_file.added_rows_count.is_some()
+        && manifest_file.existing_rows_count.is_some()
+        && manifest_file.deleted_rows_count.is_some()
+}
+
+async fn populate_retained_manifest_counts(
+    store_ctx: &StoreContext,
+    manifest_file: &mut crate::spec::manifest_list::ManifestFile,
+) -> Result<(), String> {
+    if manifest_counts_are_complete(manifest_file) {
+        return Ok(());
+    }
+
+    let manifest = crate::io::load_manifest(store_ctx, manifest_file.manifest_path.as_str())
+        .await
+        .map_err(|error| {
+            format!(
+                "failed to load retained manifest {} to recover missing counts: {error}",
+                manifest_file.manifest_path
+            )
+        })?;
+    let mut file_counts = [0i32; 3];
+    let mut row_counts = [0i64; 3];
+    for entry in manifest.entries() {
+        let index = match entry.status {
+            ManifestStatus::Added => 0,
+            ManifestStatus::Existing => 1,
+            ManifestStatus::Deleted => 2,
+        };
+        file_counts[index] = file_counts[index]
+            .checked_add(1)
+            .ok_or_else(|| "Iceberg manifest file count overflow".to_string())?;
+        let record_count = i64::try_from(entry.data_file.record_count)
+            .map_err(|_| "Iceberg manifest row count overflow".to_string())?;
+        row_counts[index] = row_counts[index]
+            .checked_add(record_count)
+            .ok_or_else(|| "Iceberg manifest row count overflow".to_string())?;
+    }
+
+    manifest_file
+        .added_files_count
+        .get_or_insert(file_counts[0]);
+    manifest_file
+        .existing_files_count
+        .get_or_insert(file_counts[1]);
+    manifest_file
+        .deleted_files_count
+        .get_or_insert(file_counts[2]);
+    manifest_file.added_rows_count.get_or_insert(row_counts[0]);
+    manifest_file
+        .existing_rows_count
+        .get_or_insert(row_counts[1]);
+    manifest_file
+        .deleted_rows_count
+        .get_or_insert(row_counts[2]);
+    Ok(())
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub enum SnapshotUpdateKind {
+    FastAppend,
+    FullOverwrite,
+    RowDelta,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct SnapshotChanges {
+    added_data_files: usize,
+    added_delete_files: usize,
+}
+
+impl SnapshotChanges {
+    fn from_added_files(added_data_files: &[DataFile], added_delete_files: &[DataFile]) -> Self {
+        Self {
+            added_data_files: added_data_files.len(),
+            added_delete_files: added_delete_files.len(),
+        }
+    }
+
+    fn adds_data_files(self) -> bool {
+        self.added_data_files > 0
+    }
+
+    fn adds_delete_files(self) -> bool {
+        self.added_delete_files > 0
+    }
+}
+
+impl SnapshotUpdateKind {
+    fn summary_operation(self, changes: SnapshotChanges) -> Operation {
+        match self {
+            Self::FastAppend => Operation::Append,
+            Self::FullOverwrite => Operation::Overwrite,
+            Self::RowDelta if changes.adds_data_files() && !changes.adds_delete_files() => {
+                Operation::Append
+            }
+            Self::RowDelta if changes.adds_delete_files() && !changes.adds_data_files() => {
+                Operation::Delete
+            }
+            Self::RowDelta => Operation::Overwrite,
+        }
+    }
+
+    fn carries_parent_manifests(self) -> bool {
+        matches!(self, Self::FastAppend | Self::RowDelta)
+    }
+
+    fn delete_totals_base<'a>(
+        self,
+        parent_summary: &'a crate::spec::snapshots::Summary,
+        is_bootstrap: bool,
+    ) -> DeleteRowTotalsBase<'a> {
+        if is_bootstrap || !self.carries_parent_manifests() {
+            DeleteRowTotalsBase::Empty
+        } else {
+            DeleteRowTotalsBase::Parent(parent_summary)
+        }
+    }
 }
 
 pub struct SnapshotProducer<'a> {
     pub tx: &'a Transaction,
     pub added_data_files: Vec,
+    pub added_delete_files: Vec,
     pub store_ctx: Option,
     pub manifest_metadata: Option,
+    pub partition_specs: Vec,
     pub write_path_mode: crate::utils::WritePathMode,
     /// If true, create a snapshot with no parent (for bootstrap scenarios)
     pub is_bootstrap: bool,
@@ -49,8 +348,10 @@ impl<'a> SnapshotProducer<'a> {
         Self {
             tx,
             added_data_files,
+            added_delete_files: Vec::new(),
             store_ctx,
             manifest_metadata,
+            partition_specs: Vec::new(),
             write_path_mode: crate::utils::WritePathMode::Absolute,
             is_bootstrap: false,
             row_lineage_start_row_id: None,
@@ -74,41 +375,105 @@ impl<'a> SnapshotProducer<'a> {
         self
     }
 
+    pub fn with_added_delete_files(mut self, delete_files: Vec) -> Self {
+        self.added_delete_files = delete_files;
+        self
+    }
+
+    pub fn with_partition_specs(mut self, partition_specs: Vec) -> Self {
+        self.partition_specs = partition_specs;
+        self
+    }
+
     pub fn validate_added_data_files(&self, _files: &[DataFile]) -> Result<(), String> {
         // TODO: Implement this function to validate the added data files
         Ok(())
     }
 
-    pub async fn commit(self, op: impl SnapshotProduceOperation) -> Result {
+    pub async fn commit(self, update_kind: SnapshotUpdateKind) -> Result {
         let timestamp_ms = crate::utils::timestamp::monotonic_timestamp_ms();
-        let is_overwrite = op.operation() == Operation::Overwrite.as_str();
-        let summary = if is_overwrite {
-            crate::spec::snapshots::Summary::new(Operation::Overwrite)
-        } else {
-            crate::spec::snapshots::Summary::new(Operation::Append)
-        };
+        let changes =
+            SnapshotChanges::from_added_files(&self.added_data_files, &self.added_delete_files);
+        let operation = update_kind.summary_operation(changes);
+        let added_data_file_count = self.added_data_files.len();
+        let added_records = self
+            .added_data_files
+            .iter()
+            .map(|df| df.record_count)
+            .sum::();
+        let mut added_position_delete_files = 0usize;
+        let mut added_position_deletes = 0u64;
+        let mut added_equality_delete_files = 0usize;
+        let mut added_equality_deletes = 0u64;
+        for df in &self.added_delete_files {
+            match df.content {
+                crate::spec::DataContentType::PositionDeletes => {
+                    added_position_delete_files += 1;
+                    added_position_deletes += df.record_count;
+                }
+                crate::spec::DataContentType::EqualityDeletes => {
+                    added_equality_delete_files += 1;
+                    added_equality_deletes += df.record_count;
+                }
+                crate::spec::DataContentType::Data => {}
+            }
+        }
+        let mut summary = crate::spec::snapshots::Summary::new(operation.clone());
+        if added_data_file_count > 0 {
+            summary = summary
+                .with_property("added-data-files", added_data_file_count.to_string())
+                .with_property("added-records", added_records.to_string());
+        }
+        if !self.added_delete_files.is_empty() {
+            summary = summary.with_property(
+                "added-delete-files",
+                self.added_delete_files.len().to_string(),
+            );
+            if added_position_delete_files > 0 {
+                summary = summary
+                    .with_property(
+                        "added-position-delete-files",
+                        added_position_delete_files.to_string(),
+                    )
+                    .with_property("added-position-deletes", added_position_deletes.to_string());
+            }
+            if added_equality_delete_files > 0 {
+                summary = summary
+                    .with_property(
+                        "added-equality-delete-files",
+                        added_equality_delete_files.to_string(),
+                    )
+                    .with_property("added-equality-deletes", added_equality_deletes.to_string());
+            }
+        }
 
         // Build manifest metadata: prefer caller-provided metadata derived from table schema/spec
         // Fall back to deriving from the current transaction snapshot if not provided
-        let metadata = if let Some(meta) = self.manifest_metadata.clone() {
-            meta
-        } else {
-            let schema_id = self.tx.snapshot().schema_id().unwrap_or_default();
-            let schema = Schema::builder()
-                .with_schema_id(schema_id)
-                .with_fields(vec![])
-                .build()
-                .map_err(|e| format!("schema build error: {e}"))?;
-            let partition_spec = PartitionSpec::builder().with_spec_id(0).build();
-            crate::spec::manifest::ManifestMetadata::new(
-                std::sync::Arc::new(schema.clone()),
-                schema_id,
-                partition_spec,
-                FormatVersion::V2,
-                ManifestContentType::Data,
-            )
+        let metadata = match self.manifest_metadata.clone() {
+            Some(meta) => meta,
+            _ => {
+                let schema_id = self.tx.snapshot().schema_id().unwrap_or_default();
+                let schema = Schema::builder()
+                    .with_schema_id(schema_id)
+                    .with_fields(vec![])
+                    .build()
+                    .map_err(|e| format!("schema build error: {e}"))?;
+                let partition_spec = PartitionSpec::builder().with_spec_id(0).build();
+                crate::spec::manifest::ManifestMetadata::new(
+                    std::sync::Arc::new(schema.clone()),
+                    schema_id,
+                    partition_spec,
+                    FormatVersion::V2,
+                    ManifestContentType::Data,
+                )
+            }
         };
         let format_version = metadata.format_version;
+        validate_delete_files_for_format(format_version, &self.added_delete_files)?;
+        let data_manifest_inputs =
+            data_manifest_inputs(&metadata, &self.partition_specs, &self.added_data_files)?;
+        let delete_manifest_inputs =
+            delete_manifest_inputs(&metadata, &self.partition_specs, &self.added_delete_files)?;
 
         let store_ctx = self
             .store_ctx
@@ -117,24 +482,23 @@ impl<'a> SnapshotProducer<'a> {
 
         // Generate new snapshot ID using UUID (not timestamp) and sequence number
         let new_snapshot_id = crate::utils::snapshot_id::generate_snapshot_id();
-        let new_sequence_number = if self.is_bootstrap {
-            1 // First snapshot starts at sequence 1
-        } else {
-            self.tx.snapshot().sequence_number() + 1
-        };
+        let new_sequence_number = self.tx.next_sequence_number()?;
 
         let parent_snapshot = self.tx.snapshot();
         let parent_manifest_list_path_str = parent_snapshot.manifest_list();
         let mut parent_manifest_entries = Vec::new();
 
-        if !self.is_bootstrap && !is_overwrite && !parent_manifest_list_path_str.is_empty() {
+        if !self.is_bootstrap
+            && update_kind.carries_parent_manifests()
+            && !parent_manifest_list_path_str.is_empty()
+        {
             let (store_ref, manifest_list_path) = store_ctx
                 .resolve(parent_manifest_list_path_str)
                 .map_err(|e| format!("{}", e))?;
 
             log::trace!(
                 "snapshot producer: loading parent manifest list: {}",
-                &manifest_list_path
+                manifest_list_path
             );
             let manifest_list_data = store_ref
                 .get(&manifest_list_path)
@@ -149,7 +513,10 @@ impl<'a> SnapshotProducer<'a> {
                 "snapshot producer: found parent manifest files: {}",
                 parent_manifest_list.entries().len()
             );
-            parent_manifest_entries.extend(parent_manifest_list.entries().iter().cloned());
+            for mut manifest_file in parent_manifest_list.into_entries() {
+                populate_retained_manifest_counts(store_ctx, &mut manifest_file).await?;
+                parent_manifest_entries.push(manifest_file);
+            }
         }
 
         let new_added_rows: i64 = self
@@ -174,49 +541,110 @@ impl<'a> SnapshotProducer<'a> {
             }
         }
 
-        let new_manifest_first_row_id = row_lineage_next_row_id;
         if self.row_lineage_start_row_id.is_some() {
             snapshot_added_rows += new_added_rows;
         }
 
-        let mut writer = ManifestWriterBuilder::new(None, None, metadata.clone()).build();
-        let added_data_files = self.added_data_files.clone();
-        for df in &added_data_files {
-            writer.add(df.clone());
-        }
-        let manifest = writer.finish();
-        let manifest_bytes = manifest.to_avro_bytes_v2()?;
+        let mut new_manifest_files = Vec::new();
+        for (data_metadata, added_data_files) in data_manifest_inputs {
+            let manifest_first_row_id = row_lineage_next_row_id;
+            let manifest_added_rows = added_data_files
+                .iter()
+                .map(|file| file.record_count as i64)
+                .sum::();
+            if let Some(next_row_id) = &mut row_lineage_next_row_id {
+                *next_row_id += manifest_added_rows;
+            }
+            let mut writer = ManifestWriterBuilder::new(None, None, data_metadata.clone()).build();
+            for df in &added_data_files {
+                writer.add(df.clone());
+            }
+            let manifest = writer.finish();
+            let manifest_bytes = manifest.to_avro_bytes_v2()?;
 
-        let manifest_len = manifest_bytes.len() as i64;
-        let manifest_rel = format!("metadata/manifest-{}.avro", uuid::Uuid::new_v4());
-        let manifest_path = object_store::path::Path::from(manifest_rel.as_str());
-        store_ctx
-            .prefixed
-            .put(
-                &manifest_path,
-                object_store::PutPayload::from(Bytes::from(manifest_bytes)),
-            )
-            .await
-            .map_err(|e| format!("{}", e))?;
+            let manifest_len = manifest_bytes.len() as i64;
+            let manifest_rel = format!("metadata/manifest-{}.avro", uuid::Uuid::new_v4());
+            let manifest_path = object_store::path::Path::from(manifest_rel.as_str());
+            store_ctx
+                .prefixed
+                .put(
+                    &manifest_path,
+                    object_store::PutPayload::from(Bytes::from(manifest_bytes)),
+                )
+                .await
+                .map_err(|e| format!("{}", e))?;
 
-        let mut manifest_file_builder = crate::spec::manifest_list::ManifestFile::builder()
-            .with_manifest_path(join_table_uri(
-                self.tx.table_uri(),
-                &manifest_rel,
-                &self.write_path_mode,
-            ))
-            .with_manifest_length(manifest_len)
-            .with_partition_spec_id(metadata.partition_spec.spec_id())
-            .with_content(ManifestContentType::Data)
-            .with_sequence_number(new_sequence_number)
-            .with_min_sequence_number(new_sequence_number)
-            .with_added_snapshot_id(new_snapshot_id)
-            .with_file_counts(added_data_files.len() as i32, 0, 0)
-            .with_row_counts(new_added_rows, 0, 0);
-        if let Some(first_row_id) = new_manifest_first_row_id {
-            manifest_file_builder = manifest_file_builder.with_first_row_id(first_row_id);
+            let mut manifest_file_builder = crate::spec::manifest_list::ManifestFile::builder()
+                .with_manifest_path(join_table_uri(
+                    self.tx.table_uri(),
+                    &manifest_rel,
+                    &self.write_path_mode,
+                ))
+                .with_manifest_length(manifest_len)
+                .with_partition_spec_id(data_metadata.partition_spec.spec_id())
+                .with_content(ManifestContentType::Data)
+                .with_sequence_number(new_sequence_number)
+                .with_min_sequence_number(new_sequence_number)
+                .with_added_snapshot_id(new_snapshot_id)
+                .with_file_counts(added_data_files.len() as i32, 0, 0)
+                .with_row_counts(manifest_added_rows, 0, 0);
+            if let Some(first_row_id) = manifest_first_row_id {
+                manifest_file_builder = manifest_file_builder.with_first_row_id(first_row_id);
+            }
+            new_manifest_files.push(manifest_file_builder.build()?);
         }
-        let manifest_file = manifest_file_builder.build()?;
+
+        for (delete_metadata, added_delete_files) in delete_manifest_inputs {
+            let mut writer =
+                ManifestWriterBuilder::new(None, None, delete_metadata.clone()).build();
+            for df in &added_delete_files {
+                writer.add(df.clone());
+            }
+            let manifest = writer.finish();
+            let manifest_bytes = manifest.to_avro_bytes_v2()?;
+            let manifest_len = manifest_bytes.len() as i64;
+            let manifest_rel = format!("metadata/manifest-{}.avro", uuid::Uuid::new_v4());
+            let manifest_path = object_store::path::Path::from(manifest_rel.as_str());
+            store_ctx
+                .prefixed
+                .put(
+                    &manifest_path,
+                    object_store::PutPayload::from(Bytes::from(manifest_bytes)),
+                )
+                .await
+                .map_err(|e| format!("{}", e))?;
+            let added_delete_rows = added_delete_files
+                .iter()
+                .map(|df| df.record_count as i64)
+                .sum::();
+            new_manifest_files.push(
+                crate::spec::manifest_list::ManifestFile::builder()
+                    .with_manifest_path(join_table_uri(
+                        self.tx.table_uri(),
+                        &manifest_rel,
+                        &self.write_path_mode,
+                    ))
+                    .with_manifest_length(manifest_len)
+                    .with_partition_spec_id(delete_metadata.partition_spec.spec_id())
+                    .with_content(ManifestContentType::Deletes)
+                    .with_sequence_number(new_sequence_number)
+                    .with_min_sequence_number(new_sequence_number)
+                    .with_added_snapshot_id(new_snapshot_id)
+                    .with_file_counts(added_delete_files.len() as i32, 0, 0)
+                    .with_row_counts(added_delete_rows, 0, 0)
+                    .build()?,
+            );
+        }
+
+        summary = with_manifest_totals(
+            summary,
+            update_kind.delete_totals_base(parent_snapshot.summary(), self.is_bootstrap),
+            parent_manifest_entries
+                .iter()
+                .chain(new_manifest_files.iter()),
+            added_position_deletes,
+            added_equality_deletes,
+        );
 
         let mut list_writer = ManifestListWriter::new();
         let mut total_manifest_count = 0;
@@ -233,8 +661,10 @@ impl<'a> SnapshotProducer<'a> {
             self.tx.snapshot().snapshot_id()
         );
 
-        list_writer.append(manifest_file);
-        total_manifest_count += 1;
+        for manifest_file in new_manifest_files {
+            list_writer.append(manifest_file);
+            total_manifest_count += 1;
+        }
         log::trace!(
             "snapshot producer: new manifest list will have files: {}",
             total_manifest_count
@@ -315,3 +745,457 @@ impl<'a> SnapshotProducer<'a> {
         Ok(ActionCommit::new(updates, requirements))
     }
 }
+
+#[cfg(test)]
+#[expect(clippy::expect_used)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use super::*;
+    use crate::spec::manifest::ManifestMetadata;
+    use crate::spec::manifest_list::ManifestFile;
+    use crate::spec::types::values::{Literal, PrimitiveLiteral};
+    use crate::spec::types::{NestedField, PrimitiveType, Type};
+    use crate::spec::{DataContentType, DataFileFormat, Transform};
+
+    fn nullable_manifest_list_bytes(manifest_path: &str, manifest_length: i64) -> Vec {
+        #[derive(serde::Serialize)]
+        struct NullableManifestCounts<'a> {
+            manifest_path: &'a str,
+            manifest_length: i64,
+            partition_spec_id: i32,
+            added_snapshot_id: i64,
+            added_data_files_count: Option,
+            existing_data_files_count: Option,
+            deleted_data_files_count: Option,
+            added_rows_count: Option,
+            existing_rows_count: Option,
+            deleted_rows_count: Option,
+        }
+
+        let schema = apache_avro::Schema::parse_str(
+            r#"{
+              "type": "record",
+              "name": "manifest_file",
+              "fields": [
+                {"name": "manifest_path", "type": "string"},
+                {"name": "manifest_length", "type": "long"},
+                {"name": "partition_spec_id", "type": "int"},
+                {"name": "added_snapshot_id", "type": "long"},
+                {"name": "added_data_files_count", "type": ["null", "int"], "default": null},
+                {"name": "existing_data_files_count", "type": ["null", "int"], "default": null},
+                {"name": "deleted_data_files_count", "type": ["null", "int"], "default": null},
+                {"name": "added_rows_count", "type": ["null", "long"], "default": null},
+                {"name": "existing_rows_count", "type": ["null", "long"], "default": null},
+                {"name": "deleted_rows_count", "type": ["null", "long"], "default": null}
+              ]
+            }"#,
+        )
+        .expect("nullable manifest list schema");
+        let mut writer = apache_avro::Writer::new(&schema, Vec::new());
+        writer
+            .append_ser(NullableManifestCounts {
+                manifest_path,
+                manifest_length,
+                partition_spec_id: 2,
+                added_snapshot_id: 7,
+                added_data_files_count: None,
+                existing_data_files_count: None,
+                deleted_data_files_count: None,
+                added_rows_count: None,
+                existing_rows_count: None,
+                deleted_rows_count: None,
+            })
+            .expect("nullable manifest list entry");
+        writer.into_inner().expect("nullable manifest list bytes")
+    }
+
+    fn manifest_file(content: ManifestContentType) -> ManifestFile {
+        ManifestFile::builder()
+            .with_manifest_path("metadata/manifest.avro")
+            .with_content(content)
+            .build()
+            .expect("manifest file")
+    }
+
+    fn delete_file(path: &str, partition_spec_id: i32) -> DataFile {
+        DataFile {
+            content: DataContentType::PositionDeletes,
+            file_path: path.to_string(),
+            file_format: DataFileFormat::Parquet,
+            partition: vec![],
+            record_count: 1,
+            file_size_in_bytes: 1,
+            column_sizes: HashMap::new(),
+            value_counts: HashMap::new(),
+            null_value_counts: HashMap::new(),
+            nan_value_counts: HashMap::new(),
+            lower_bounds: HashMap::new(),
+            upper_bounds: HashMap::new(),
+            block_size_in_bytes: None,
+            key_metadata: None,
+            split_offsets: vec![],
+            equality_ids: vec![],
+            sort_order_id: None,
+            first_row_id: None,
+            partition_spec_id,
+            referenced_data_file: Some("data.parquet".to_string()),
+            content_offset: None,
+            content_size_in_bytes: None,
+        }
+    }
+
+    #[test]
+    fn manifest_totals_omit_only_totals_with_unknown_active_counts() {
+        let data_with_unknown_counts = manifest_file(ManifestContentType::Data);
+        let deletes_with_unknown_files = manifest_file(ManifestContentType::Deletes);
+        let summary = with_manifest_totals(
+            crate::spec::snapshots::Summary::new(Operation::Append),
+            DeleteRowTotalsBase::Parent(&crate::spec::snapshots::Summary::new(Operation::Append)),
+            [&data_with_unknown_counts, &deletes_with_unknown_files],
+            0,
+            0,
+        );
+
+        assert!(
+            !summary
+                .additional_properties
+                .contains_key("total-data-files")
+        );
+        assert!(!summary.additional_properties.contains_key("total-records"));
+        assert!(
+            !summary
+                .additional_properties
+                .contains_key("total-delete-files")
+        );
+    }
+
+    #[test]
+    fn full_overwrite_recomputes_snapshot_totals_from_new_manifests() {
+        let parent_summary = crate::spec::snapshots::Summary::new(Operation::Append)
+            .with_property("total-position-deletes", "7")
+            .with_property("total-equality-deletes", "5");
+        let replacement_data = ManifestFile::builder()
+            .with_manifest_path("metadata/replacement.avro")
+            .with_content(ManifestContentType::Data)
+            .with_file_counts(2, 0, 0)
+            .with_row_counts(10, 0, 0)
+            .build()
+            .expect("replacement manifest");
+
+        let summary = with_manifest_totals(
+            crate::spec::snapshots::Summary::new(Operation::Overwrite),
+            SnapshotUpdateKind::FullOverwrite.delete_totals_base(&parent_summary, false),
+            [&replacement_data],
+            0,
+            0,
+        );
+
+        let totals = &summary.additional_properties;
+        assert_eq!(
+            totals.get("total-data-files").map(String::as_str),
+            Some("2")
+        );
+        assert_eq!(
+            totals.get("total-delete-files").map(String::as_str),
+            Some("0")
+        );
+        assert_eq!(totals.get("total-records").map(String::as_str), Some("10"));
+        assert_eq!(
+            totals.get("total-position-deletes").map(String::as_str),
+            Some("0")
+        );
+        assert_eq!(
+            totals.get("total-equality-deletes").map(String::as_str),
+            Some("0")
+        );
+    }
+
+    #[test]
+    fn delete_manifest_inputs_group_files_by_historical_partition_spec() {
+        let schema = Schema::builder().build().expect("schema");
+        let current_spec = PartitionSpec::builder().with_spec_id(2).build();
+        let historical_spec = PartitionSpec::builder().with_spec_id(1).build();
+        let metadata = ManifestMetadata::new(
+            Arc::new(schema),
+            0,
+            current_spec.clone(),
+            FormatVersion::V2,
+            ManifestContentType::Data,
+        );
+        let files = vec![
+            delete_file("delete-1-a.parquet", 1),
+            delete_file("delete-2.parquet", 2),
+            delete_file("delete-1-b.parquet", 1),
+        ];
+
+        let groups = delete_manifest_inputs(&metadata, &[historical_spec], &files)
+            .expect("partition specs should resolve");
+
+        assert_eq!(groups.len(), 2);
+        assert_eq!(groups[0].0.partition_spec.spec_id(), 1);
+        assert_eq!(groups[0].0.content, ManifestContentType::Deletes);
+        assert_eq!(groups[0].1.len(), 2);
+        assert!(groups[0].1.iter().all(|file| file.partition_spec_id == 1));
+        assert_eq!(groups[1].0.partition_spec.spec_id(), 2);
+        assert_eq!(groups[1].0.content, ManifestContentType::Deletes);
+        assert_eq!(groups[1].1.len(), 1);
+        assert!(groups[1].1.iter().all(|file| file.partition_spec_id == 2));
+    }
+
+    #[test]
+    fn delete_manifest_inputs_reject_missing_partition_spec() {
+        let metadata = ManifestMetadata::new(
+            Arc::new(Schema::builder().build().expect("schema")),
+            0,
+            PartitionSpec::builder().with_spec_id(2).build(),
+            FormatVersion::V2,
+            ManifestContentType::Data,
+        );
+
+        let error = delete_manifest_inputs(&metadata, &[], &[delete_file("delete.parquet", 1)])
+            .expect_err("unknown partition spec must fail");
+
+        assert!(error.contains("partition spec 1 is missing"));
+    }
+
+    #[test]
+    fn snapshot_commit_rejects_position_deletes_after_format_v3_upgrade() {
+        futures::executor::block_on(async {
+            let table_url =
+                url::Url::parse("file:///tmp/iceberg-v3-position-delete/").expect("table URL");
+            let store: Arc =
+                Arc::new(object_store::memory::InMemory::new());
+            let store_ctx = StoreContext::new(store, &table_url).expect("store context");
+            let schema = Schema::builder().build().expect("schema");
+            let partition_spec = PartitionSpec::builder().with_spec_id(0).build();
+            let metadata = ManifestMetadata::new(
+                Arc::new(schema),
+                0,
+                partition_spec.clone(),
+                FormatVersion::V3,
+                ManifestContentType::Data,
+            );
+            let parent_snapshot = SnapshotBuilder::new()
+                .with_snapshot_id(0)
+                .with_sequence_number(0)
+                .with_manifest_list(String::new())
+                .with_summary(crate::spec::snapshots::Summary::new(Operation::Append))
+                .build()
+                .expect("parent snapshot");
+            let transaction = Transaction::new(table_url.to_string(), parent_snapshot, 0);
+
+            let result =
+                SnapshotProducer::new(&transaction, vec![], Some(store_ctx), Some(metadata))
+                    .with_bootstrap(true)
+                    .with_added_delete_files(vec![delete_file("delete.parquet", 0)])
+                    .with_partition_specs(vec![partition_spec])
+                    .commit(SnapshotUpdateKind::RowDelta)
+                    .await;
+
+            let error = result
+                .err()
+                .expect("v3 snapshot commit must reject position delete files");
+            assert!(error.contains("v3"));
+            assert!(error.contains("position delete"));
+        });
+    }
+
+    #[test]
+    fn snapshot_commit_uses_data_files_historical_partition_spec() {
+        futures::executor::block_on(async {
+            let table_url = url::Url::parse("file:///tmp/iceberg-concurrent-spec-evolution/")
+                .expect("table URL");
+            let store: Arc =
+                Arc::new(object_store::memory::InMemory::new());
+            let store_ctx = StoreContext::new(store, &table_url).expect("store context");
+            let schema = Schema::builder().build().expect("schema");
+            let historical_spec = PartitionSpec::builder().with_spec_id(1).build();
+            let current_spec = PartitionSpec::builder().with_spec_id(2).build();
+            let metadata = ManifestMetadata::new(
+                Arc::new(schema),
+                0,
+                current_spec.clone(),
+                FormatVersion::V2,
+                ManifestContentType::Data,
+            );
+            let parent_snapshot = SnapshotBuilder::new()
+                .with_snapshot_id(0)
+                .with_sequence_number(0)
+                .with_manifest_list(String::new())
+                .with_summary(crate::spec::snapshots::Summary::new(Operation::Append))
+                .build()
+                .expect("parent snapshot");
+            let transaction = Transaction::new(table_url.to_string(), parent_snapshot, 0);
+            let mut added_data_file = delete_file("data.parquet", historical_spec.spec_id());
+            added_data_file.content = DataContentType::Data;
+            added_data_file.referenced_data_file = None;
+
+            let action_commit = SnapshotProducer::new(
+                &transaction,
+                vec![added_data_file],
+                Some(store_ctx.clone()),
+                Some(metadata),
+            )
+            .with_bootstrap(true)
+            .with_partition_specs(vec![historical_spec, current_spec])
+            .commit(SnapshotUpdateKind::RowDelta)
+            .await
+            .expect("snapshot commit");
+            let snapshot = action_commit
+                .updates()
+                .iter()
+                .find_map(|update| match update {
+                    TableUpdate::AddSnapshot { snapshot } => Some(snapshot),
+                    _ => None,
+                })
+                .expect("added snapshot");
+            let manifest_list = crate::io::load_manifest_list(&store_ctx, snapshot.manifest_list())
+                .await
+                .expect("manifest list");
+            let added_data_manifest = manifest_list
+                .entries()
+                .iter()
+                .find(|manifest| manifest.content == ManifestContentType::Data)
+                .expect("data manifest");
+
+            assert_eq!(added_data_manifest.partition_spec_id, 1);
+        });
+    }
+
+    #[test]
+    fn snapshot_producer_writes_historical_delete_manifests_and_table_sequence() {
+        futures::executor::block_on(async {
+            let table_url = url::Url::parse("file:///tmp/iceberg-table/").expect("table URL");
+            let store: Arc =
+                Arc::new(object_store::memory::InMemory::new());
+            let store_ctx = StoreContext::new(store, &table_url).expect("store context");
+            let schema = Schema::builder()
+                .with_fields(vec![Arc::new(NestedField::required(
+                    1,
+                    "part",
+                    Type::Primitive(PrimitiveType::Int),
+                ))])
+                .build()
+                .expect("schema");
+            let historical_spec = PartitionSpec::builder()
+                .with_spec_id(1)
+                .add_field_with_id(1, 1000, "part", Transform::Identity)
+                .build();
+            let current_spec = PartitionSpec::builder().with_spec_id(2).build();
+            let metadata = ManifestMetadata::new(
+                Arc::new(schema),
+                0,
+                current_spec.clone(),
+                FormatVersion::V2,
+                ManifestContentType::Data,
+            );
+            let mut parent_manifest_writer =
+                ManifestWriterBuilder::new(None, None, metadata.clone()).build();
+            let mut parent_data_file = delete_file("data.parquet", 2);
+            parent_data_file.content = DataContentType::Data;
+            parent_data_file.referenced_data_file = None;
+            parent_manifest_writer.add(parent_data_file);
+            let parent_manifest_bytes = parent_manifest_writer
+                .finish()
+                .to_avro_bytes_v2()
+                .expect("parent manifest bytes");
+            store_ctx
+                .prefixed
+                .put(
+                    &object_store::path::Path::from("metadata/parent-manifest.avro"),
+                    object_store::PutPayload::from(Bytes::from(parent_manifest_bytes.clone())),
+                )
+                .await
+                .expect("parent manifest");
+            let parent_list_bytes = nullable_manifest_list_bytes(
+                "metadata/parent-manifest.avro",
+                parent_manifest_bytes.len() as i64,
+            );
+            store_ctx
+                .prefixed
+                .put(
+                    &object_store::path::Path::from("metadata/parent-list.avro"),
+                    object_store::PutPayload::from(Bytes::from(parent_list_bytes)),
+                )
+                .await
+                .expect("parent manifest list");
+            let parent_snapshot = SnapshotBuilder::new()
+                .with_snapshot_id(7)
+                .with_sequence_number(3)
+                .with_manifest_list("metadata/parent-list.avro".to_string())
+                .with_summary(crate::spec::snapshots::Summary::new(Operation::Append))
+                .build()
+                .expect("parent snapshot");
+            let transaction = Transaction::new(table_url.to_string(), parent_snapshot, 11);
+            let mut historical_delete = delete_file("delete-1.parquet", 1);
+            historical_delete.partition = vec![Some(Literal::Primitive(PrimitiveLiteral::Int(42)))];
+            let action_commit = SnapshotProducer::new(
+                &transaction,
+                vec![],
+                Some(store_ctx.clone()),
+                Some(metadata),
+            )
+            .with_added_delete_files(vec![historical_delete, delete_file("delete-2.parquet", 2)])
+            .with_partition_specs(vec![historical_spec, current_spec])
+            .commit(SnapshotUpdateKind::RowDelta)
+            .await
+            .expect("row delta snapshot");
+            let snapshot = action_commit
+                .updates()
+                .iter()
+                .find_map(|update| match update {
+                    TableUpdate::AddSnapshot { snapshot } => Some(snapshot),
+                    _ => None,
+                })
+                .expect("added snapshot");
+
+            assert_eq!(snapshot.sequence_number(), 12);
+            assert_eq!(
+                snapshot
+                    .summary()
+                    .additional_properties
+                    .get("total-data-files"),
+                Some(&"1".to_string())
+            );
+            assert_eq!(
+                snapshot
+                    .summary()
+                    .additional_properties
+                    .get("total-records"),
+                Some(&"1".to_string())
+            );
+            let manifest_list = crate::io::load_manifest_list(&store_ctx, snapshot.manifest_list())
+                .await
+                .expect("manifest list");
+            let delete_manifests = manifest_list
+                .entries()
+                .iter()
+                .filter(|manifest| manifest.content == ManifestContentType::Deletes)
+                .collect::>();
+            assert_eq!(delete_manifests.len(), 2);
+            for manifest_file in delete_manifests {
+                let manifest =
+                    crate::io::load_manifest(&store_ctx, manifest_file.manifest_path.as_str())
+                        .await
+                        .expect("delete manifest");
+                assert_eq!(
+                    manifest.metadata().partition_spec.spec_id(),
+                    manifest_file.partition_spec_id
+                );
+                assert_eq!(manifest.metadata().content, ManifestContentType::Deletes);
+                assert!(manifest.entries().iter().all(|entry| {
+                    entry.data_file.partition_spec_id == manifest_file.partition_spec_id
+                }));
+                if manifest_file.partition_spec_id == 1 {
+                    assert_eq!(manifest.metadata().partition_spec.fields().len(), 1);
+                    assert_eq!(
+                        manifest.entries()[0].data_file.partition,
+                        vec![Some(Literal::Primitive(PrimitiveLiteral::Int(42)))]
+                    );
+                }
+            }
+        });
+    }
+}
diff --git a/crates/sail-iceberg/src/operations/write/arrow_parquet.rs b/crates/sail-iceberg/src/operations/write/arrow_parquet.rs
index 9f800a269..caefbca51 100644
--- a/crates/sail-iceberg/src/operations/write/arrow_parquet.rs
+++ b/crates/sail-iceberg/src/operations/write/arrow_parquet.rs
@@ -24,6 +24,7 @@ pub struct ParquetFileMeta {
 }
 
 pub struct ArrowParquetWriter {
+    // FIXME: Stream Parquet output to object storage instead of retaining the full file in memory.
     writer: Option>>,
 }
 
diff --git a/crates/sail-iceberg/src/operations/write/base_writer/data_file_writer.rs b/crates/sail-iceberg/src/operations/write/base_writer/data_file_writer.rs
index 4e7296ce6..278340970 100644
--- a/crates/sail-iceberg/src/operations/write/base_writer/data_file_writer.rs
+++ b/crates/sail-iceberg/src/operations/write/base_writer/data_file_writer.rs
@@ -12,8 +12,8 @@
 
 use std::collections::HashMap;
 
-use crate::operations::write::arrow_parquet::ParquetFileMeta;
 use crate::operations::write::WriteOutcome;
+use crate::operations::write::arrow_parquet::ParquetFileMeta;
 use crate::spec::types::values::Literal;
 use crate::spec::{DataContentType, DataFile, DataFileFormat, Datum};
 
diff --git a/crates/sail-iceberg/src/operations/write/config.rs b/crates/sail-iceberg/src/operations/write/config.rs
index dbea74797..7075cac7d 100644
--- a/crates/sail-iceberg/src/operations/write/config.rs
+++ b/crates/sail-iceberg/src/operations/write/config.rs
@@ -17,8 +17,8 @@ use parquet::file::properties::WriterProperties;
 use sail_common_datafusion::catalog::CatalogPartitionField;
 pub use sail_common_datafusion::variant::VariantShreddingConfig;
 
-use crate::spec::partition::UnboundPartitionSpec;
 use crate::spec::Schema as IcebergSchema;
+use crate::spec::partition::UnboundPartitionSpec;
 
 #[derive(Debug, Clone)]
 pub struct WriterConfig {
diff --git a/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs b/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs
index 06800df38..fc5dab415 100644
--- a/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs
+++ b/crates/sail-iceberg/src/operations/write/file_writer/location_generator.rs
@@ -22,7 +22,6 @@ pub trait LocationGenerator {
 
 pub struct DefaultLocationGenerator {
     base: ObjectPath,
-    data_dir: String,
     counter: AtomicU64,
 }
 
@@ -30,15 +29,6 @@ impl DefaultLocationGenerator {
     pub fn new(base: ObjectPath) -> Self {
         Self {
             base,
-            data_dir: "data".to_string(),
-            counter: AtomicU64::new(0),
-        }
-    }
-
-    pub fn new_with_data_dir(base: ObjectPath, data_dir: String) -> Self {
-        Self {
-            base,
-            data_dir: data_dir.trim_matches('/').to_string(),
             counter: AtomicU64::new(0),
         }
     }
@@ -53,10 +43,8 @@ impl LocationGenerator for DefaultLocationGenerator {
         let id = self.counter.fetch_add(1, Ordering::Relaxed);
         let file = format!("part-{}-{:020}.parquet", Uuid::new_v4(), id);
         let rel_unencoded = match partition_dir {
-            Some(dir) if !dir.is_empty() => {
-                format!("{}/{}/{}", self.data_dir, dir.trim_matches('/'), file)
-            }
-            _ => format!("{}/{}", self.data_dir, file),
+            Some(dir) if !dir.is_empty() => format!("{}/{}", dir.trim_matches('/'), file),
+            _ => file,
         };
         // Join each component to avoid encoding '/' into '%2F'
         let mut full = self.base.clone();
diff --git a/crates/sail-iceberg/src/operations/write/partition.rs b/crates/sail-iceberg/src/operations/write/partition.rs
index d6a793601..a09bb734f 100644
--- a/crates/sail-iceberg/src/operations/write/partition.rs
+++ b/crates/sail-iceberg/src/operations/write/partition.rs
@@ -195,11 +195,7 @@ pub fn split_record_batch_by_partition(
     }
 
     use std::collections::HashMap;
-    struct Group {
-        values: Vec>,
-        indices: Vec,
-    }
-    let mut groups: HashMap = HashMap::new();
+    let mut groups: HashMap>, Vec> = HashMap::new();
 
     let num_rows = batch.num_rows();
     for row in 0..num_rows {
@@ -218,22 +214,18 @@ pub fn split_record_batch_by_partition(
                 .unwrap_or(&Type::Primitive(PrimitiveType::String));
             vals.push(apply_transform(f.transform, field_type, lit));
         }
-        let dir = build_partition_dir(spec, iceberg_schema, &vals)?;
-        let entry = groups.entry(dir).or_insert_with(|| Group {
-            values: vals.clone(),
-            indices: Vec::new(),
-        });
-        entry.indices.push(row as u32);
+        groups.entry(vals).or_default().push(row as u32);
     }
 
     let mut out: Vec = Vec::with_capacity(groups.len());
-    for (dir, grp) in groups.into_iter() {
-        let indices = UInt32Array::from(grp.indices);
+    for (partition_values, group_indices) in groups {
+        let partition_dir = build_partition_dir(spec, iceberg_schema, &partition_values)?;
+        let indices = UInt32Array::from(group_indices);
         let rb = compute::take_record_batch(batch, &indices).map_err(|e| e.to_string())?;
         out.push(PartitionBatchResult {
             record_batch: rb,
-            partition_values: grp.values,
-            partition_dir: dir,
+            partition_values,
+            partition_dir,
             spec_id: 0,
         });
     }
diff --git a/crates/sail-iceberg/src/operations/write/table_writer.rs b/crates/sail-iceberg/src/operations/write/table_writer.rs
index 9092a6989..3faec6247 100644
--- a/crates/sail-iceberg/src/operations/write/table_writer.rs
+++ b/crates/sail-iceberg/src/operations/write/table_writer.rs
@@ -13,12 +13,12 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use datafusion::arrow::array::{new_null_array, ArrayRef};
+use datafusion::arrow::array::{ArrayRef, new_null_array};
 use datafusion::arrow::datatypes::{FieldRef, Schema, SchemaRef};
 use datafusion::arrow::record_batch::RecordBatch;
 use datafusion_common::{DataFusionError, Result};
-use object_store::path::Path as ObjectPath;
 use object_store::ObjectStoreExt;
+use object_store::path::Path as ObjectPath;
 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
 use sail_common_datafusion::array::record_batch::cast_record_batch_relaxed_tz;
 use url::Url;
@@ -31,13 +31,13 @@ use crate::operations::write::file_writer::location_generator::{
 };
 use crate::operations::write::partition::split_record_batch_by_partition;
 use crate::operations::write::variant_shredding::{
-    apply_variant_shredding_plan, build_variant_shredding_plan,
-    unshred_shredded_variants_for_write, VariantShreddingPlan,
+    VariantShreddingPlan, apply_variant_shredding_plan, build_variant_shredding_plan,
+    unshred_shredded_variants_for_write,
 };
+use crate::spec::DataFile;
 use crate::spec::schema::Schema as IcebergSchema;
-use crate::spec::types::values::Literal;
 use crate::spec::types::NestedField;
-use crate::spec::DataFile;
+use crate::spec::types::values::Literal;
 use crate::utils::conversions::to_scalar;
 
 enum PartitionWriterState {
@@ -51,15 +51,18 @@ enum PartitionWriterState {
     },
 }
 
+struct PartitionWriter {
+    partition_dir: String,
+    state: PartitionWriterState,
+}
+
 pub struct IcebergTableWriter {
     pub store: Arc,
     pub config: WriterConfig,
     pub generator: DefaultLocationGenerator,
-    pub table_url: Url,
-    // partition_dir -> writer
-    writers: HashMap,
-    // partition_dir -> partition values aligned with spec
-    partition_values_map: HashMap>>,
+    pub data_url: Url,
+    // Typed partition tuple -> writer.
+    writers: HashMap>, PartitionWriter>,
     written: Vec,
     pub partition_spec_id: i32,
 }
@@ -70,16 +73,14 @@ impl IcebergTableWriter {
         root: ObjectPath,
         config: WriterConfig,
         partition_spec_id: i32,
-        data_dir: String,
-        table_url: Url,
+        data_url: Url,
     ) -> Self {
         Self {
-            generator: DefaultLocationGenerator::new_with_data_dir(root, data_dir),
+            generator: DefaultLocationGenerator::new(root),
             store,
             config,
-            table_url,
+            data_url,
             writers: HashMap::new(),
-            partition_values_map: HashMap::new(),
             written: Vec::new(),
             partition_spec_id,
         }
@@ -96,9 +97,7 @@ impl IcebergTableWriter {
         if spec.fields.is_empty() {
             // Unpartitioned: write as-is once
             let partition_dir = String::new();
-            self.partition_values_map
-                .entry(partition_dir.clone())
-                .or_default();
+            let partition_values = Vec::new();
             let padded = Self::align_batch_with_table_schema(
                 batch,
                 &self.config.table_schema,
@@ -109,16 +108,15 @@ impl IcebergTableWriter {
                 unshred_shredded_variants_for_write(&padded, &self.config.table_schema)?;
             let aligned = cast_record_batch_relaxed_tz(&normalized, &self.config.table_schema)
                 .map_err(|e| e.to_string())?;
-            self.write_aligned_batch(partition_dir, aligned).await?;
+            self.write_aligned_batch(partition_values, partition_dir, aligned)
+                .await?;
             return Ok(());
         }
 
         let parts = split_record_batch_by_partition(batch, spec, iceberg_schema)?;
         for p in parts.into_iter() {
             let partition_dir = p.partition_dir;
-            self.partition_values_map
-                .entry(partition_dir.clone())
-                .or_insert(p.partition_values);
+            let partition_values = p.partition_values;
             let padded = Self::align_batch_with_table_schema(
                 &p.record_batch,
                 &self.config.table_schema,
@@ -129,7 +127,8 @@ impl IcebergTableWriter {
                 unshred_shredded_variants_for_write(&padded, &self.config.table_schema)?;
             let aligned = cast_record_batch_relaxed_tz(&normalized, &self.config.table_schema)
                 .map_err(|e| e.to_string())?;
-            self.write_aligned_batch(partition_dir, aligned).await?;
+            self.write_aligned_batch(partition_values, partition_dir, aligned)
+                .await?;
         }
 
         Ok(())
@@ -137,16 +136,22 @@ impl IcebergTableWriter {
 
     async fn write_aligned_batch(
         &mut self,
+        partition_values: Vec>,
         partition_dir: String,
         batch: RecordBatch,
     ) -> Result<(), String> {
-        let state = self
-            .writers
-            .remove(&partition_dir)
-            .map(Ok)
-            .unwrap_or_else(|| self.new_partition_writer_state())?;
+        let (partition_dir, state) = match self.writers.remove(&partition_values) {
+            Some(writer) => (writer.partition_dir, writer.state),
+            None => (partition_dir, self.new_partition_writer_state()?),
+        };
         let state = self.write_partition_state(state, batch).await?;
-        self.writers.insert(partition_dir, state);
+        self.writers.insert(
+            partition_values,
+            PartitionWriter {
+                partition_dir,
+                state,
+            },
+        );
         Ok(())
     }
 
@@ -266,49 +271,43 @@ impl IcebergTableWriter {
         }
     }
 
-    pub async fn flush_partition(
+    async fn flush_partition(
         &mut self,
+        state: PartitionWriterState,
         partition_dir: &str,
         partition_values: Vec>,
     ) -> Result<(), String> {
-        if let Some(state) = self.writers.remove(partition_dir) {
-            let writer = self.finish_partition_state(state).await?;
-            let (bytes, meta) = writer.close().await?;
-            let (rel, full) = self.generator.with_partition_dir(Some(partition_dir));
-            log::trace!("iceberg.table_writer.flush_partition.writing: {}", &full);
-            self.store
-                .put(&full, object_store::PutPayload::from(bytes))
-                .await
-                .map_err(|e| e.to_string())?;
-            log::trace!(
-                "iceberg.table_writer.flush_partition.written: rel={} full={}",
-                &rel,
-                &full
-            );
-            let file_path = match self.table_url.join(&rel) {
-                Ok(u) => u.to_string(),
-                Err(_) => {
-                    format!("{}{}", self.table_url.as_str(), rel)
-                }
-            };
-            let df = DataFileWriter::new(self.partition_spec_id, file_path, partition_values)
-                .finish(meta)?
-                .data_file;
-            self.written.push(df);
-        }
+        let writer = self.finish_partition_state(state).await?;
+        let (bytes, meta) = writer.close().await?;
+        let (rel, full) = self.generator.with_partition_dir(Some(partition_dir));
+        log::trace!("iceberg.table_writer.flush_partition.writing: {}", full);
+        self.store
+            .put(&full, object_store::PutPayload::from(bytes))
+            .await
+            .map_err(|e| e.to_string())?;
+        log::trace!(
+            "iceberg.table_writer.flush_partition.written: rel={} full={}",
+            rel,
+            full
+        );
+        // Prevent a leading partition segment containing ':' from being parsed as a URI scheme.
+        let file_path = match self.data_url.join(&format!("./{rel}")) {
+            Ok(u) => u.to_string(),
+            Err(_) => {
+                format!("{}{}", self.data_url.as_str(), rel)
+            }
+        };
+        let df = DataFileWriter::new(self.partition_spec_id, file_path, partition_values)
+            .finish(meta)?
+            .data_file;
+        self.written.push(df);
         Ok(())
     }
 
     pub async fn close(mut self) -> Result, String> {
-        let keys: Vec = self.writers.keys().cloned().collect();
-        for k in keys {
-            let vals = self
-                .partition_values_map
-                .remove(&k)
-                .unwrap_or_default()
-                .into_iter()
-                .collect();
-            self.flush_partition(&k, vals).await?;
+        for (partition_values, writer) in std::mem::take(&mut self.writers) {
+            self.flush_partition(writer.state, &writer.partition_dir, partition_values)
+                .await?;
         }
         Ok(self.written)
     }
diff --git a/crates/sail-iceberg/src/operations/write/variant_shredding.rs b/crates/sail-iceberg/src/operations/write/variant_shredding.rs
index 84eb10146..7d43d458c 100644
--- a/crates/sail-iceberg/src/operations/write/variant_shredding.rs
+++ b/crates/sail-iceberg/src/operations/write/variant_shredding.rs
@@ -1,7 +1,6 @@
 pub use sail_common_datafusion::variant::{
-    apply_variant_shredding_plan, build_variant_shredding_plan,
-    unshred_shredded_variants_for_write, VariantShreddingPlan,
-    DEFAULT_VARIANT_INFERENCE_NODE_BUDGET,
+    DEFAULT_VARIANT_INFERENCE_NODE_BUDGET, VariantShreddingPlan, apply_variant_shredding_plan,
+    build_variant_shredding_plan, unshred_shredded_variants_for_write,
 };
 
 #[cfg(test)]
@@ -14,7 +13,7 @@ mod tests {
     use datafusion_common::{DataFusionError, Result};
     use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
     use parquet_variant_compute::{
-        json_to_variant, shred_variant, unshred_variant, variant_to_json, VariantArray, VariantType,
+        VariantArray, VariantType, json_to_variant, shred_variant, unshred_variant, variant_to_json,
     };
     use sail_common_datafusion::array::record_batch::cast_record_batch_relaxed_tz;
 
@@ -159,9 +158,11 @@ mod tests {
         let DataType::Struct(payload_fields) = payload_field.data_type() else {
             return Err(DataFusionError::Plan("expected variant struct".to_string()));
         };
-        assert!(payload_fields
-            .iter()
-            .any(|field| field.name() == "typed_value"));
+        assert!(
+            payload_fields
+                .iter()
+                .any(|field| field.name() == "typed_value")
+        );
         let typed_value = payload_fields
             .iter()
             .find(|field| field.name() == "typed_value")
diff --git a/crates/sail-iceberg/src/options.rs b/crates/sail-iceberg/src/options.rs
index d83714652..090c914a1 100644
--- a/crates/sail-iceberg/src/options.rs
+++ b/crates/sail-iceberg/src/options.rs
@@ -13,18 +13,18 @@
 use datafusion::catalog::Session;
 use sail_common_datafusion::datasource::OptionLayer;
 pub(crate) use sail_data_source::options::{
-    parsers, BuildPartialOptions, PartialOptions, ResolveOptions,
+    BuildPartialOptions, PartialOptions, ResolveOptions, parsers,
 };
 
 use crate::error::DataSourceResult;
 
-pub mod gen {
+pub mod r#gen {
     include!(concat!(env!("OUT_DIR"), "/options/iceberg.rs"));
 }
 
-impl ResolveOptions for gen::IcebergReadOptions {
+impl ResolveOptions for r#gen::IcebergReadOptions {
     fn resolve(_ctx: &dyn Session, options: Vec) -> DataSourceResult {
-        let mut partial = gen::IcebergReadPartialOptions::initialize();
+        let mut partial = r#gen::IcebergReadPartialOptions::initialize();
         for layer in options {
             partial.merge(layer.build_partial_options()?);
         }
@@ -32,9 +32,9 @@ impl ResolveOptions for gen::IcebergReadOptions {
     }
 }
 
-impl ResolveOptions for gen::IcebergWriteOptions {
+impl ResolveOptions for r#gen::IcebergWriteOptions {
     fn resolve(_ctx: &dyn Session, options: Vec) -> DataSourceResult {
-        let mut partial = gen::IcebergWritePartialOptions::initialize();
+        let mut partial = r#gen::IcebergWritePartialOptions::initialize();
         for layer in options {
             partial.merge(layer.build_partial_options()?);
         }
diff --git a/crates/sail-iceberg/src/physical/mod.rs b/crates/sail-iceberg/src/physical/mod.rs
index d91bdda60..58968daba 100644
--- a/crates/sail-iceberg/src/physical/mod.rs
+++ b/crates/sail-iceberg/src/physical/mod.rs
@@ -1,3 +1,4 @@
+mod row_level_planner;
 pub mod table_scan_planner;
 
 pub use table_scan_planner::IcebergPhysicalPlanner;
diff --git a/crates/sail-iceberg/src/physical/row_level_planner.rs b/crates/sail-iceberg/src/physical/row_level_planner.rs
new file mode 100644
index 000000000..2dc888414
--- /dev/null
+++ b/crates/sail-iceberg/src/physical/row_level_planner.rs
@@ -0,0 +1,184 @@
+use std::sync::Arc;
+
+use datafusion::common::{DataFusionError, Result, not_impl_err, plan_err};
+use datafusion::execution::SessionState;
+use datafusion::logical_expr::logical_plan::builder::LogicalPlanBuilder;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::physical_planner::PhysicalPlanner;
+use sail_common_datafusion::datasource::{PhysicalSinkMode, RowLevelCommand};
+use sail_data_source::options::ResolveOptions;
+use sail_logical_plan::merge::RowLevelWriteNode;
+
+use crate::operations::SnapshotUpdateKind;
+use crate::options::r#gen::IcebergWriteOptions;
+use crate::physical_plan::merge_row_projection::IcebergMergeRowProjection;
+use crate::physical_plan::{
+    IcebergCommitExec, IcebergEqualityDeleteWriterExec, IcebergWriterExec, IcebergWriterExecOptions,
+};
+use crate::table::Table;
+use crate::table_format::{
+    IcebergTableFormat, catalog_managed_iceberg_from_options, metadata_location_from_options,
+    split_iceberg_write_options_and_table_properties,
+};
+
+pub(crate) async fn plan_iceberg_row_level_write(
+    session_state: &SessionState,
+    planner: &dyn PhysicalPlanner,
+    node: &RowLevelWriteNode,
+    physical_inputs: &[Arc],
+) -> Result> {
+    match node.command() {
+        RowLevelCommand::Delete => plan_iceberg_delete(session_state, planner, node).await,
+        RowLevelCommand::Merge => plan_iceberg_merge(session_state, node, physical_inputs).await,
+        command => not_impl_err!("Iceberg row-level {command:?} operations"),
+    }
+}
+
+async fn plan_iceberg_merge(
+    session_state: &SessionState,
+    node: &RowLevelWriteNode,
+    physical_inputs: &[Arc],
+) -> Result> {
+    let write_plan = physical_inputs.first().cloned().ok_or_else(|| {
+        DataFusionError::Internal("Iceberg MERGE missing write plan input".to_string())
+    })?;
+    if node.touched_files_plan().is_some() && physical_inputs.len() < 2 {
+        return plan_err!("Iceberg MERGE missing touched-file plan input");
+    }
+    let table_url =
+        IcebergTableFormat::parse_table_url(vec![node.target_location().to_string()]).await?;
+    let metadata_location = metadata_location_from_options(node.target_options());
+    let catalog_managed_table = catalog_managed_iceberg_from_options(node.target_options());
+    let metadata_location_for_load = catalog_managed_table
+        .then_some(metadata_location.clone())
+        .flatten();
+    let table = Table::load_with_metadata_location(
+        session_state,
+        table_url.clone(),
+        metadata_location_for_load,
+    )
+    .await?;
+    ensure_current_row_level_mode(&table, RowLevelCommand::Merge)?;
+    let partition_columns = IcebergTableFormat::partition_columns_from_metadata(&table)?;
+    let writer_options = resolve_row_level_writer_options(session_state, node)?;
+
+    let merge_projection = IcebergMergeRowProjection::try_new(write_plan.schema())?;
+    let data_rows_schema = merge_projection.data_schema();
+    let writer_input: Arc = Arc::new(CoalescePartitionsExec::new(write_plan));
+    let writer: Arc = Arc::new(IcebergWriterExec::new_merge(
+        writer_input,
+        table_url.clone(),
+        partition_columns,
+        PhysicalSinkMode::Append,
+        true,
+        writer_options.clone(),
+        Some(data_rows_schema),
+    ));
+
+    Ok(Arc::new(
+        IcebergCommitExec::new(
+            Arc::new(CoalescePartitionsExec::new(writer)),
+            table_url,
+            writer_options.lakehouse_table.clone(),
+            SnapshotUpdateKind::RowDelta,
+        )
+        .with_expected_snapshot_id(node.expected_snapshot_id()),
+    ))
+}
+
+async fn plan_iceberg_delete(
+    session_state: &SessionState,
+    planner: &dyn PhysicalPlanner,
+    node: &RowLevelWriteNode,
+) -> Result> {
+    // TODO: Support conditionless DELETE by scanning all rows into equality deletes.
+    let condition = node.condition().ok_or_else(|| {
+        DataFusionError::Plan(
+            "Iceberg equality-delete MOR DELETE requires a WHERE condition".to_string(),
+        )
+    })?;
+
+    let table_url =
+        IcebergTableFormat::parse_table_url(vec![node.target_location().to_string()]).await?;
+    let metadata_location = metadata_location_from_options(node.target_options());
+    let catalog_managed_table = catalog_managed_iceberg_from_options(node.target_options());
+    let metadata_location_for_load = catalog_managed_table.then_some(metadata_location).flatten();
+    let table = Table::load_with_metadata_location(
+        session_state,
+        table_url.clone(),
+        metadata_location_for_load,
+    )
+    .await?;
+    ensure_current_row_level_mode(&table, RowLevelCommand::Delete)?;
+
+    let delete_plan = LogicalPlanBuilder::from(node.raw_target().as_ref().clone())
+        .filter(condition.expr.clone())?
+        .build()?;
+    let physical_delete = planner
+        .create_physical_plan(&delete_plan, session_state)
+        .await?;
+
+    let writer_options = resolve_row_level_writer_options(session_state, node)?;
+
+    let delete_input: Arc =
+        Arc::new(CoalescePartitionsExec::new(physical_delete));
+    let delete_writer: Arc = Arc::new(IcebergEqualityDeleteWriterExec::new(
+        delete_input,
+        table_url.clone(),
+        writer_options.table_properties.clone(),
+        writer_options.write_data_path.clone(),
+        writer_options.write_folder_storage_path.clone(),
+        writer_options.lakehouse_table.clone(),
+    ));
+
+    Ok(Arc::new(
+        IcebergCommitExec::new(
+            Arc::new(CoalescePartitionsExec::new(delete_writer)),
+            table_url,
+            writer_options.lakehouse_table.clone(),
+            SnapshotUpdateKind::RowDelta,
+        )
+        .with_expected_snapshot_id(node.expected_snapshot_id()),
+    ))
+}
+
+fn ensure_current_row_level_mode(table: &Table, command: RowLevelCommand) -> Result<()> {
+    let (operation, property) = match command {
+        RowLevelCommand::Delete => ("DELETE", "write.delete.mode"),
+        RowLevelCommand::Merge => ("MERGE", "write.merge.mode"),
+        RowLevelCommand::Update => ("UPDATE", "write.update.mode"),
+    };
+    let mode = table
+        .metadata()
+        .properties
+        .get(property)
+        .map_or("copy-on-write", String::as_str);
+    if mode.eq_ignore_ascii_case("merge-on-read") {
+        return Ok(());
+    }
+    if mode.eq_ignore_ascii_case("copy-on-write") {
+        return not_impl_err!(
+            "Iceberg {operation} with `{property}=copy-on-write` is not supported yet; set `{property}=merge-on-read`"
+        );
+    }
+    plan_err!(
+        "Unknown Iceberg row-level operation mode for `{property}`: {mode}; expected `copy-on-write` or `merge-on-read`"
+    )
+}
+
+fn resolve_row_level_writer_options(
+    session_state: &SessionState,
+    node: &RowLevelWriteNode,
+) -> Result {
+    let (clean_options, table_properties) =
+        split_iceberg_write_options_and_table_properties(node.target_options().to_vec())?;
+    let variant_presence =
+        IcebergWriterExecOptions::variant_shredding_option_presence(&clean_options);
+    let iceberg_options = IcebergWriteOptions::resolve(session_state, clean_options)?;
+    let mut writer_options = IcebergWriterExecOptions::from(iceberg_options);
+    writer_options.apply_variant_shredding_option_presence(variant_presence);
+    writer_options.table_properties = table_properties;
+    writer_options.lakehouse_table = node.target_lakehouse_table().cloned();
+    Ok(writer_options)
+}
diff --git a/crates/sail-iceberg/src/physical/table_scan_planner.rs b/crates/sail-iceberg/src/physical/table_scan_planner.rs
index a0f0f8ee5..7c176e140 100644
--- a/crates/sail-iceberg/src/physical/table_scan_planner.rs
+++ b/crates/sail-iceberg/src/physical/table_scan_planner.rs
@@ -8,9 +8,12 @@ use datafusion::logical_expr::expr_rewriter::unnormalize_cols;
 use datafusion::logical_expr::{LogicalPlan, TableScan, UserDefinedLogicalNode};
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner};
+use sail_logical_plan::merge::{MergeCardinalityCheckNode, RowLevelWriteNode};
+use sail_physical_plan::merge_cardinality_check::MergeCardinalityCheckExec;
 
 use crate::logical::IcebergTableSource;
-use crate::table_format::{plan_iceberg_write, IcebergWriteNode};
+use crate::physical::row_level_planner::plan_iceberg_row_level_write;
+use crate::table_format::{IcebergWriteNode, plan_iceberg_write};
 
 pub struct IcebergPhysicalPlanner;
 
@@ -18,28 +21,53 @@ pub struct IcebergPhysicalPlanner;
 impl ExtensionPlanner for IcebergPhysicalPlanner {
     async fn plan_extension(
         &self,
-        _planner: &dyn PhysicalPlanner,
+        planner: &dyn PhysicalPlanner,
         node: &dyn UserDefinedLogicalNode,
         logical_inputs: &[&LogicalPlan],
         physical_inputs: &[Arc],
         session_state: &SessionState,
     ) -> Result>> {
-        let Some(node) = node.as_any().downcast_ref::() else {
-            return Ok(None);
-        };
-        let [logical_input] = logical_inputs else {
-            return datafusion_common::internal_err!(
-                "IcebergWriteNode requires exactly one logical input"
-            );
-        };
-        let [physical_input] = physical_inputs else {
-            return datafusion_common::internal_err!(
-                "IcebergWriteNode requires exactly one physical input"
-            );
-        };
-        plan_iceberg_write(session_state, logical_input, physical_input.clone(), node)
-            .await
-            .map(Some)
+        if let Some(node) = node.as_any().downcast_ref::() {
+            let [logical_input] = logical_inputs else {
+                return datafusion_common::internal_err!(
+                    "IcebergWriteNode requires exactly one logical input"
+                );
+            };
+            let [physical_input] = physical_inputs else {
+                return datafusion_common::internal_err!(
+                    "IcebergWriteNode requires exactly one physical input"
+                );
+            };
+            return plan_iceberg_write(session_state, logical_input, physical_input.clone(), node)
+                .await
+                .map(Some);
+        }
+
+        if let Some(node) = node.as_any().downcast_ref::() {
+            let [input] = physical_inputs else {
+                return datafusion_common::internal_err!(
+                    "MergeCardinalityCheckNode requires exactly one physical input"
+                );
+            };
+            let exec = MergeCardinalityCheckExec::new(
+                Arc::clone(input),
+                node.target_row_id_col(),
+                node.target_present_col(),
+                node.source_present_col(),
+            )?;
+            return Ok(Some(Arc::new(exec)));
+        }
+
+        if let Some(node) = node.as_any().downcast_ref::() {
+            if !node.target_format().eq_ignore_ascii_case("iceberg") {
+                return Ok(None);
+            }
+            return plan_iceberg_row_level_write(session_state, planner, node, physical_inputs)
+                .await
+                .map(Some);
+        }
+
+        Ok(None)
     }
 
     async fn plan_table_scan(
diff --git a/crates/sail-iceberg/src/physical_plan/action_schema.rs b/crates/sail-iceberg/src/physical_plan/action_schema.rs
index cec53c48a..0676f3d97 100644
--- a/crates/sail-iceberg/src/physical_plan/action_schema.rs
+++ b/crates/sail-iceberg/src/physical_plan/action_schema.rs
@@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::spec::types::values::{Literal, PrimitiveLiteral};
 use crate::spec::{
-    DataContentType, DataFile, DataFileFormat, Operation, PartitionSpec, Schema as IcebergSchema,
+    DataContentType, DataFile, DataFileFormat, PartitionSpec, Schema as IcebergSchema,
     TableRequirement,
 };
 
@@ -34,7 +34,6 @@ static ACTION_SCHEMA: LazyLock =
 pub struct CommitMeta {
     pub table_uri: String,
     pub row_count: u64,
-    pub operation: Operation,
     pub requirements: Vec,
     pub table_properties: Vec<(String, String)>,
     pub lakehouse_table: Option,
@@ -46,7 +45,6 @@ pub struct CommitMeta {
 pub struct CommitMetaAction {
     pub table_uri: String,
     pub row_count: u64,
-    pub operation: String,
     /// Requirements are relatively small but hard to trace into Arrow schema; keep as JSON.
     pub requirements_json: String,
     /// Table properties are applied only when bootstrapping new table metadata.
@@ -71,6 +69,8 @@ pub enum PartitionValue {
     /// Stored as decimal string because `serde_arrow` does not support `u128` natively.
     UInt128(String),
     Binary(Vec),
+    /// The marker payload is required because `serde_arrow` cannot serialize a unit union variant.
+    Null(bool),
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -78,19 +78,25 @@ pub struct AddFileAction {
     pub content: String,
     pub file_path: String,
     pub file_format: String,
-    pub partition: Vec>,
+    pub partition: Vec,
     pub record_count: u64,
     pub file_size_in_bytes: u64,
     pub column_sizes: BTreeMap,
     pub value_counts: BTreeMap,
     pub null_value_counts: BTreeMap,
     pub split_offsets: Vec,
+    pub equality_ids: Vec,
+    pub sort_order_id: Option,
+    pub first_row_id: Option,
     pub partition_spec_id: i32,
+    pub referenced_data_file: Option,
+    pub content_offset: Option,
+    pub content_size_in_bytes: Option,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DeleteFileAction {
-    pub file_path: String,
+    pub data_file: AddFileAction,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -116,7 +122,7 @@ fn partition_value_union_type() -> DataType {
         reason = "partition_value_union_type is a process-global constant."
     )]
     let union_fields = UnionFields::try_new(
-        [0i8, 1, 2, 3, 4, 5, 6, 7, 8],
+        [0i8, 1, 2, 3, 4, 5, 6, 7, 8, 9],
         [
             Arc::new(Field::new("Boolean", DataType::Boolean, false)),
             Arc::new(Field::new("Int", DataType::Int32, false)),
@@ -131,6 +137,7 @@ fn partition_value_union_type() -> DataType {
                 DataType::List(Arc::new(Field::new("element", DataType::UInt8, false))),
                 false,
             )),
+            Arc::new(Field::new("Null", DataType::Boolean, false)),
         ],
     )
     .unwrap();
@@ -150,8 +157,8 @@ fn map_type_i32_u64() -> DataType {
     DataType::Map(entries_field, false)
 }
 
-fn iceberg_action_tracing_options(
-) -> std::result::Result {
+fn iceberg_action_tracing_options()
+-> std::result::Result {
     use serde_arrow::schema::TracingOptions;
 
     TracingOptions::default()
@@ -186,6 +193,14 @@ fn iceberg_action_tracing_options(
                 Field::new("partition", DataType::List(partition_item), false),
             )
         })
+        .and_then(|opts| {
+            let partition_item =
+                Arc::new(Field::new("element", partition_value_union_type(), false));
+            opts.overwrite(
+                "action.delete.data_file.partition",
+                Field::new("partition", DataType::List(partition_item), false),
+            )
+        })
         .map_err(|e| format!("failed to build serde_arrow tracing options: {e}"))
 }
 
@@ -204,18 +219,6 @@ pub fn iceberg_action_schema() -> Result {
     Ok(Arc::clone(&*ACTION_SCHEMA))
 }
 
-fn parse_operation(s: &str) -> Result {
-    match s {
-        "append" => Ok(Operation::Append),
-        "replace" => Ok(Operation::Replace),
-        "overwrite" => Ok(Operation::Overwrite),
-        "delete" => Ok(Operation::Delete),
-        other => Err(DataFusionError::Plan(format!(
-            "unknown iceberg operation '{other}'"
-        ))),
-    }
-}
-
 impl From for PartitionValue {
     fn from(p: PrimitiveLiteral) -> Self {
         match p {
@@ -264,6 +267,9 @@ impl TryFrom for PrimitiveLiteral {
                     })
             }
             PartitionValue::Binary(x) => Ok(PrimitiveLiteral::Binary(x)),
+            PartitionValue::Null(_) => Err(DataFusionError::Internal(
+                "null partition marker cannot be converted to a primitive literal".to_string(),
+            )),
         }
     }
 }
@@ -276,8 +282,8 @@ impl TryFrom for AddFileAction {
             .partition
             .into_iter()
             .map(|opt| match opt {
-                None => Ok(None),
-                Some(Literal::Primitive(p)) => Ok(Some(p.into())),
+                None => Ok(PartitionValue::Null(false)),
+                Some(Literal::Primitive(p)) => Ok(p.into()),
                 Some(other) => Err(DataFusionError::Internal(format!(
                     "unsupported non-primitive partition literal in DataFile: {other:?}"
                 ))),
@@ -295,7 +301,13 @@ impl TryFrom for AddFileAction {
             value_counts: df.value_counts.into_iter().collect(),
             null_value_counts: df.null_value_counts.into_iter().collect(),
             split_offsets: df.split_offsets,
+            equality_ids: df.equality_ids,
+            sort_order_id: df.sort_order_id,
+            first_row_id: df.first_row_id,
             partition_spec_id: df.partition_spec_id,
+            referenced_data_file: df.referenced_data_file,
+            content_offset: df.content_offset,
+            content_size_in_bytes: df.content_size_in_bytes,
         })
     }
 }
@@ -311,7 +323,7 @@ impl TryFrom for DataFile {
             other => {
                 return Err(DataFusionError::Plan(format!(
                     "unknown DataContentType string '{other}'"
-                )))
+                )));
             }
         };
         let file_format = match a.file_format.as_str() {
@@ -322,16 +334,16 @@ impl TryFrom for DataFile {
             other => {
                 return Err(DataFusionError::Plan(format!(
                     "unknown DataFileFormat string '{other}'"
-                )))
+                )));
             }
         };
 
         let partition = a
             .partition
             .into_iter()
-            .map(|opt| match opt {
-                None => Ok(None),
-                Some(pv) => Ok(Some(Literal::Primitive(pv.try_into()?))),
+            .map(|value| match value {
+                PartitionValue::Null(_) => Ok(None),
+                value => Ok(Some(Literal::Primitive(value.try_into()?))),
             })
             .collect::>>()?;
 
@@ -351,13 +363,13 @@ impl TryFrom for DataFile {
             block_size_in_bytes: None,
             key_metadata: None,
             split_offsets: a.split_offsets,
-            equality_ids: Vec::new(),
-            sort_order_id: None,
-            first_row_id: None,
+            equality_ids: a.equality_ids,
+            sort_order_id: a.sort_order_id,
+            first_row_id: a.first_row_id,
             partition_spec_id: a.partition_spec_id,
-            referenced_data_file: None,
-            content_offset: None,
-            content_size_in_bytes: None,
+            referenced_data_file: a.referenced_data_file,
+            content_offset: a.content_offset,
+            content_size_in_bytes: a.content_size_in_bytes,
         })
     }
 }
@@ -386,6 +398,23 @@ pub fn encode_add_data_files(data_files: Vec) -> Result {
     encode_actions(rows)
 }
 
+pub fn encode_delete_data_files(data_files: Vec) -> Result {
+    if data_files.is_empty() {
+        return Ok(RecordBatch::new_empty(iceberg_action_schema()?));
+    }
+    let rows = data_files
+        .into_iter()
+        .map(|df| {
+            Ok(ActionRow {
+                action: ExecAction::Delete(DeleteFileAction {
+                    data_file: df.try_into()?,
+                }),
+            })
+        })
+        .collect::>>()?;
+    encode_actions(rows)
+}
+
 pub fn encode_commit_meta(meta: CommitMeta) -> Result {
     let requirements_json = serde_json::to_string(&meta.requirements)
         .map_err(|e| DataFusionError::External(Box::new(e)))?;
@@ -414,7 +443,6 @@ pub fn encode_commit_meta(meta: CommitMeta) -> Result {
         action: ExecAction::CommitMeta(CommitMetaAction {
             table_uri: meta.table_uri,
             row_count: meta.row_count,
-            operation: meta.operation.as_str().to_string(),
             requirements_json,
             table_properties_json,
             lakehouse_table_json,
@@ -432,18 +460,13 @@ pub fn decode_actions_and_meta_from_batch(
         .map_err(|e| DataFusionError::External(Box::new(e)))?;
 
     let mut adds: Vec = Vec::new();
-    let deletes: Vec = Vec::new();
+    let mut deletes: Vec = Vec::new();
     let mut meta: Option = None;
 
     for row in rows {
         match row.action {
             ExecAction::Add(a) => adds.push(a.try_into()?),
-            ExecAction::Delete(_d) => {
-                // Delete files are not implemented in local commits yet; plumb through later.
-                return Err(DataFusionError::NotImplemented(
-                    "Iceberg delete file actions are not implemented".to_string(),
-                ));
-            }
+            ExecAction::Delete(d) => deletes.push(d.data_file.try_into()?),
             ExecAction::CommitMeta(m) => {
                 let requirements: Vec =
                     serde_json::from_str(&m.requirements_json)
@@ -472,7 +495,6 @@ pub fn decode_actions_and_meta_from_batch(
                 meta = Some(CommitMeta {
                     table_uri: m.table_uri,
                     row_count: m.row_count,
-                    operation: parse_operation(&m.operation)?,
                     requirements,
                     table_properties,
                     lakehouse_table,
@@ -500,7 +522,7 @@ mod tests {
             content: DataContentType::Data,
             file_path: "s3://bucket/a.parquet".to_string(),
             file_format: DataFileFormat::Parquet,
-            partition: vec![Some(Literal::Primitive(PrimitiveLiteral::Int(1)))],
+            partition: vec![Some(Literal::Primitive(PrimitiveLiteral::Int(1))), None],
             record_count: 10,
             file_size_in_bytes: 123,
             column_sizes: HashMap::from([(1, 10u64)]),
@@ -524,7 +546,6 @@ mod tests {
         let meta = CommitMeta {
             table_uri: "s3://bucket/table".to_string(),
             row_count: 10,
-            operation: Operation::Append,
             requirements: vec![TableRequirement::NotExist],
             table_properties: vec![],
             lakehouse_table: None,
@@ -535,15 +556,28 @@ mod tests {
         let schema = iceberg_action_schema()?;
         let batches = vec![
             encode_add_data_files(vec![df.clone()])?,
+            encode_delete_data_files(vec![DataFile {
+                content: DataContentType::PositionDeletes,
+                referenced_data_file: Some(df.file_path.clone()),
+                ..df.clone()
+            }])?,
             encode_commit_meta(meta)?,
         ];
         let merged = concat_batches(&schema, &batches)
             .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
 
-        let (adds, _deletes, meta) = decode_actions_and_meta_from_batch(&merged)?;
+        let (adds, deletes, meta) = decode_actions_and_meta_from_batch(&merged)?;
         assert_eq!(adds.len(), 1);
+        assert_eq!(deletes.len(), 1);
         assert_eq!(adds[0].file_path, df.file_path);
         assert_eq!(adds[0].record_count, df.record_count);
+        assert_eq!(adds[0].partition, df.partition);
+        assert_eq!(deletes[0].content, DataContentType::PositionDeletes);
+        assert_eq!(deletes[0].partition, df.partition);
+        assert_eq!(
+            deletes[0].referenced_data_file.as_deref(),
+            Some(df.file_path.as_str())
+        );
         assert!(meta.is_some());
         Ok(())
     }
diff --git a/crates/sail-iceberg/src/physical_plan/commit/commit_exec.rs b/crates/sail-iceberg/src/physical_plan/commit/commit_exec.rs
index 580265bf7..5925cb23e 100644
--- a/crates/sail-iceberg/src/physical_plan/commit/commit_exec.rs
+++ b/crates/sail-iceberg/src/physical_plan/commit/commit_exec.rs
@@ -14,8 +14,8 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use bytes::Bytes;
-use datafusion::arrow::array::UInt64Array;
-use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::arrow::array::Int64Array;
+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
 use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::execution::context::TaskContext;
 use datafusion::physical_expr::{Distribution, EquivalenceProperties};
@@ -25,25 +25,25 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
     PlanProperties, SendableRecordBatchStream,
 };
-use datafusion_common::{internal_err, DataFusionError, Result};
-use futures::stream::once;
+use datafusion_common::{DataFusionError, Result, internal_err};
 use futures::StreamExt;
+use futures::stream::once;
 use object_store::ObjectStoreExt;
 use sail_common_datafusion::catalog::LakehouseExecutionContext;
 use url::Url;
 
 use crate::catalog_support::commit::{
-    catalog_requirements, table_metadata_location, CatalogCommitOutcome, CatalogTableInfo,
-    IcebergCatalogCommitCoordinator, IcebergCatalogCommitMode,
+    CatalogCommitOutcome, CatalogTableInfo, IcebergCatalogCommitCoordinator,
+    IcebergCatalogCommitMode, catalog_requirements, table_metadata_location,
 };
 use crate::io::StoreContext;
 use crate::operations::bootstrap::{
-    bootstrap_first_snapshot, bootstrap_new_table_with_style, bootstrap_snapshot_action_commit,
-    NewTableMetadataStyle, PersistStrategy,
+    NewTableMetadataStyle, PersistStrategy, bootstrap_first_snapshot,
+    bootstrap_new_table_with_style, bootstrap_snapshot_action_commit,
 };
 use crate::operations::helpers::format_version_for_schema;
-use crate::operations::{SnapshotProduceOperation, Transaction, TransactionAction};
-use crate::physical_plan::action_schema::decode_actions_and_meta_from_batch;
+use crate::operations::{SnapshotUpdateKind, Transaction, TransactionAction};
+use crate::physical_plan::action_schema::{CommitMeta, decode_actions_and_meta_from_batch};
 use crate::physical_plan::commit::IcebergCommitInfo;
 use crate::spec::catalog::TableUpdate;
 use crate::spec::metadata::table_metadata::SnapshotLog;
@@ -59,11 +59,31 @@ use crate::utils::get_object_store_from_context;
 use crate::utils::metadata::metadata_files_for_version;
 const MAX_COMMIT_RETRIES: usize = 5;
 
+fn commit_count_batch(schema: SchemaRef, row_count: u64) -> Result {
+    let row_count = i64::try_from(row_count).map_err(|e| {
+        DataFusionError::Execution(format!("Iceberg commit row count overflow: {e}"))
+    })?;
+    let array = Arc::new(Int64Array::from(vec![row_count]));
+    RecordBatch::try_new(schema, vec![array])
+        .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+}
+
+fn expected_snapshot_requirement(
+    expected_snapshot_id: Option>,
+) -> Option {
+    expected_snapshot_id.map(|snapshot_id| TableRequirement::RefSnapshotIdMatch {
+        r#ref: MAIN_BRANCH.to_string(),
+        snapshot_id,
+    })
+}
+
 #[derive(Debug)]
 pub struct IcebergCommitExec {
     input: Arc,
     table_url: Url,
     lakehouse_table: Option,
+    snapshot_update_kind: SnapshotUpdateKind,
+    expected_snapshot_id: Option>,
     cache: Arc,
 }
 
@@ -72,10 +92,11 @@ impl IcebergCommitExec {
         input: Arc,
         table_url: Url,
         lakehouse_table: Option,
+        snapshot_update_kind: SnapshotUpdateKind,
     ) -> Self {
         let schema = Arc::new(Schema::new(vec![Field::new(
             "count",
-            DataType::UInt64,
+            DataType::Int64,
             true,
         )]));
         let cache = Arc::new(PlanProperties::new(
@@ -88,10 +109,17 @@ impl IcebergCommitExec {
             input,
             table_url,
             lakehouse_table,
+            snapshot_update_kind,
+            expected_snapshot_id: None,
             cache,
         }
     }
 
+    pub fn with_expected_snapshot_id(mut self, expected_snapshot_id: Option>) -> Self {
+        self.expected_snapshot_id = expected_snapshot_id;
+        self
+    }
+
     pub fn table_url(&self) -> &Url {
         &self.table_url
     }
@@ -104,6 +132,14 @@ impl IcebergCommitExec {
         self.lakehouse_table.as_ref()
     }
 
+    pub fn snapshot_update_kind(&self) -> SnapshotUpdateKind {
+        self.snapshot_update_kind
+    }
+
+    pub fn expected_snapshot_id(&self) -> Option> {
+        self.expected_snapshot_id
+    }
+
     fn apply_schema_update(table_meta: &mut TableMetadata, new_schema: IcebergSchema) {
         let schema_id = new_schema.schema_id();
         let highest_field_id = new_schema.highest_field_id();
@@ -141,9 +177,9 @@ impl IcebergCommitExec {
             table_meta.partition_specs.push(new_spec.clone());
         }
         table_meta.default_spec_id = spec_id;
-        if let Some(highest) = new_spec.highest_field_id() {
-            table_meta.last_partition_id = table_meta.last_partition_id.max(highest);
-        }
+        table_meta.last_partition_id = table_meta
+            .last_partition_id
+            .max(new_spec.last_assigned_field_id());
     }
 
     fn validate_requirements(
@@ -340,6 +376,33 @@ impl IcebergCommitExec {
     fn table_metadata_location(table_url: &Url, metadata_file: &str) -> Result {
         table_metadata_location(table_url, metadata_file)
     }
+
+    fn merge_writer_commit_meta(
+        accumulated: &mut Option,
+        mut incoming: CommitMeta,
+    ) -> Result<()> {
+        let Some(existing) = accumulated.as_mut() else {
+            *accumulated = Some(incoming);
+            return Ok(());
+        };
+
+        let incoming_row_count = incoming.row_count;
+        incoming.row_count = existing.row_count;
+        if existing != &incoming {
+            return Err(DataFusionError::Internal(
+                "inconsistent commit_meta actions from Iceberg writer partitions".to_string(),
+            ));
+        }
+        existing.row_count = existing
+            .row_count
+            .checked_add(incoming_row_count)
+            .ok_or_else(|| {
+                DataFusionError::Execution(
+                    "Iceberg writer row count overflow across partitions".to_string(),
+                )
+            })?;
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -367,11 +430,15 @@ impl ExecutionPlan for IcebergCommitExec {
         if children.len() != 1 {
             return internal_err!("IcebergCommitExec requires exactly one child");
         }
-        Ok(Arc::new(Self::new(
-            Arc::clone(&children[0]),
-            self.table_url.clone(),
-            self.lakehouse_table.clone(),
-        )))
+        Ok(Arc::new(
+            Self::new(
+                Arc::clone(&children[0]),
+                self.table_url.clone(),
+                self.lakehouse_table.clone(),
+                self.snapshot_update_kind,
+            )
+            .with_expected_snapshot_id(self.expected_snapshot_id),
+        ))
     }
 
     fn execute(
@@ -394,32 +461,37 @@ impl ExecutionPlan for IcebergCommitExec {
 
         let table_url = self.table_url.clone();
         let lakehouse_table = self.lakehouse_table.clone();
+        let snapshot_update_kind = self.snapshot_update_kind;
+        let expected_snapshot_id = self.expected_snapshot_id;
         let schema = self.schema();
         let future = async move {
             let object_store = get_object_store_from_context(&context, &table_url)?;
             let store_ctx = StoreContext::new(object_store.clone(), &table_url)?;
 
             // Read writer result as Arrow-native action batches (may be empty for IgnoreIfExists).
+            // FIXME: Track task-produced files so failed or cancelled commits can remove
+            // uncommitted files.
             let mut data = input_stream;
             let mut added_data_files = Vec::new();
+            let mut added_delete_files = Vec::new();
             let mut commit_meta = None;
             while let Some(batch_result) = data.next().await {
                 let batch = batch_result?;
                 if batch.num_rows() == 0 {
                     continue;
                 }
-                let (adds, _deletes, meta) = decode_actions_and_meta_from_batch(&batch)?;
+                let (adds, deletes, meta) = decode_actions_and_meta_from_batch(&batch)?;
                 added_data_files.extend(adds);
-                if meta.is_some() {
-                    commit_meta = meta;
+                added_delete_files.extend(deletes);
+                if let Some(meta) = meta {
+                    Self::merge_writer_commit_meta(&mut commit_meta, meta)?;
                 }
             }
 
             // No-op path (e.g. IgnoreIfExists on existing table): no rows, no meta.
-            if commit_meta.is_none() && added_data_files.is_empty() {
-                let array = Arc::new(UInt64Array::from(vec![0u64]));
-                let batch = RecordBatch::try_new(schema, vec![array])?;
-                return Ok(batch);
+            if commit_meta.is_none() && added_data_files.is_empty() && added_delete_files.is_empty()
+            {
+                return commit_count_batch(schema, 0);
             }
 
             let commit_meta = commit_meta.ok_or_else(|| {
@@ -428,20 +500,26 @@ impl ExecutionPlan for IcebergCommitExec {
                 )
             })?;
 
-            let commit_info = IcebergCommitInfo {
+            let mut commit_info = IcebergCommitInfo {
                 table_uri: commit_meta.table_uri,
                 row_count: commit_meta.row_count,
                 data_files: added_data_files,
+                delete_files: added_delete_files,
                 manifest_path: String::new(),
                 manifest_list_path: String::new(),
                 updates: vec![],
                 requirements: commit_meta.requirements,
                 table_properties: commit_meta.table_properties,
                 lakehouse_table: commit_meta.lakehouse_table.or(lakehouse_table),
-                operation: commit_meta.operation,
+                snapshot_update_kind,
                 schema: commit_meta.schema,
                 partition_spec: commit_meta.partition_spec,
             };
+            if let Some(requirement) = expected_snapshot_requirement(expected_snapshot_id)
+                && !commit_info.requirements.contains(&requirement)
+            {
+                commit_info.requirements.push(requirement);
+            }
 
             let catalog_table = commit_info
                 .lakehouse_table
@@ -499,10 +577,7 @@ impl ExecutionPlan for IcebergCommitExec {
                 catalog_commit_mode
             );
 
-            if latest_meta_res.is_err()
-                && (matches!(commit_info.operation, crate::spec::Operation::Overwrite)
-                    || matches!(commit_info.operation, crate::spec::Operation::Append))
-            {
+            if latest_meta_res.is_err() {
                 Self::validate_requirements(None, &commit_info.requirements)?;
                 if let Some(catalog_table) = catalog_metadata_update_table {
                     let bootstrap_result = bootstrap_new_table_with_style(
@@ -551,9 +626,7 @@ impl ExecutionPlan for IcebergCommitExec {
                     }
                 }
 
-                let array = Arc::new(UInt64Array::from(vec![commit_info.row_count]));
-                let batch = RecordBatch::try_new(schema, vec![array])?;
-                return Ok(batch);
+                return commit_count_batch(schema, commit_info.row_count);
             }
 
             let initial_latest_meta = latest_meta_res?;
@@ -645,10 +718,7 @@ impl ExecutionPlan for IcebergCommitExec {
 
                 // If metadata exists but there is no current snapshot (e.g. from a CREATE TABLE),
                 // bootstrap the first snapshot as a normal metadata version.
-                if maybe_snapshot.is_none()
-                    && (matches!(commit_info.operation, crate::spec::Operation::Overwrite)
-                        || matches!(commit_info.operation, crate::spec::Operation::Append))
-                {
+                if maybe_snapshot.is_none() {
                     let mut catalog_fallback_table = catalog_metadata_update_table;
                     if let Some(catalog_table) = catalog_commit_table {
                         let action_commit = bootstrap_snapshot_action_commit(
@@ -690,10 +760,7 @@ impl ExecutionPlan for IcebergCommitExec {
                                 if committed.payload().is_some() {
                                     log::trace!("Iceberg catalog commit returned a payload");
                                 }
-                                let array =
-                                    Arc::new(UInt64Array::from(vec![commit_info.row_count]));
-                                let batch = RecordBatch::try_new(schema, vec![array])?;
-                                return Ok(batch);
+                                return commit_count_batch(schema, commit_info.row_count);
                             }
                             CatalogCommitOutcome::NotSupported => {
                                 if matches!(
@@ -766,9 +833,7 @@ impl ExecutionPlan for IcebergCommitExec {
                         .await?;
                     }
 
-                    let array = Arc::new(UInt64Array::from(vec![commit_info.row_count]));
-                    let batch = RecordBatch::try_new(schema, vec![array])?;
-                    return Ok(batch);
+                    return commit_count_batch(schema, commit_info.row_count);
                 }
 
                 let snapshot = maybe_snapshot.ok_or_else(|| {
@@ -793,19 +858,24 @@ impl ExecutionPlan for IcebergCommitExec {
                     continue;
                 }
 
-                // Build transaction and action based on operation
-                let tx = Transaction::new(table_url.to_string(), snapshot);
+                // Build transaction and action based on the snapshot update algorithm.
+                let tx = Transaction::new(
+                    table_url.to_string(),
+                    snapshot,
+                    table_meta.last_sequence_number,
+                );
                 let manifest_meta = tx.default_manifest_metadata(
                     &schema_iceberg,
                     &partition_spec_for_commit,
                     table_meta.format_version,
                 );
-                let action_commit = match commit_info.operation {
-                    crate::spec::Operation::Append => {
+                let action_commit = match commit_info.snapshot_update_kind {
+                    SnapshotUpdateKind::FastAppend => {
                         let mut action = tx
                             .fast_append()
                             .with_store_context(store_ctx.clone())
                             .with_manifest_metadata(manifest_meta)
+                            .with_partition_specs(table_meta.partition_specs.clone())
                             .with_row_lineage_start_row_id(row_lineage_start_row_id);
                         for df in commit_info.data_files.clone().into_iter() {
                             action.add_file(df);
@@ -815,30 +885,22 @@ impl ExecutionPlan for IcebergCommitExec {
                             .await
                             .map_err(DataFusionError::Execution)?
                     }
-                    crate::spec::Operation::Overwrite => {
+                    update_kind @ (SnapshotUpdateKind::FullOverwrite
+                    | SnapshotUpdateKind::RowDelta) => {
                         let producer = crate::operations::SnapshotProducer::new(
                             &tx,
                             commit_info.data_files.clone(),
                             Some(store_ctx.clone()),
                             Some(manifest_meta),
                         )
+                        .with_added_delete_files(commit_info.delete_files.clone())
+                        .with_partition_specs(table_meta.partition_specs.clone())
                         .with_row_lineage_start_row_id(row_lineage_start_row_id);
-                        struct LocalOverwriteOperation;
-                        impl SnapshotProduceOperation for LocalOverwriteOperation {
-                            fn operation(&self) -> &'static str {
-                                "overwrite"
-                            }
-                        }
                         producer
-                            .commit(LocalOverwriteOperation)
+                            .commit(update_kind)
                             .await
                             .map_err(DataFusionError::Execution)?
                     }
-                    _ => {
-                        return Err(DataFusionError::NotImplemented(
-                            "Unsupported Iceberg operation in commit".to_string(),
-                        ));
-                    }
                 };
 
                 // Apply updates (only handle the ones we emit: AddSnapshot, SetSnapshotRef)
@@ -875,9 +937,7 @@ impl ExecutionPlan for IcebergCommitExec {
                             if committed.payload().is_some() {
                                 log::trace!("Iceberg catalog commit returned a payload");
                             }
-                            let array = Arc::new(UInt64Array::from(vec![commit_info.row_count]));
-                            let batch = RecordBatch::try_new(schema, vec![array])?;
-                            return Ok(batch);
+                            return commit_count_batch(schema, commit_info.row_count);
                         }
                         CatalogCommitOutcome::NotSupported
                             if matches!(
@@ -899,9 +959,10 @@ impl ExecutionPlan for IcebergCommitExec {
                     }
                 }
 
-                log::trace!("commit_exec: applying updates: {:?}", &action_updates);
+                log::trace!("commit_exec: applying updates: {:?}", action_updates);
                 let mut newest_snapshot_seq: Option = None;
                 let mut newest_snapshot_added_rows: Option = None;
+                let previous_metadata_timestamp_ms = table_meta.last_updated_ms;
                 let timestamp_ms = crate::utils::timestamp::monotonic_timestamp_ms();
                 for upd in action_updates {
                     match upd {
@@ -924,10 +985,10 @@ impl ExecutionPlan for IcebergCommitExec {
                         _ => {}
                     }
                 }
-                if let Some(seq) = newest_snapshot_seq {
-                    if seq > table_meta.last_sequence_number {
-                        table_meta.last_sequence_number = seq;
-                    }
+                if let Some(seq) = newest_snapshot_seq
+                    && seq > table_meta.last_sequence_number
+                {
+                    table_meta.last_sequence_number = seq;
                 }
                 table_meta.last_updated_ms = timestamp_ms;
                 if let Some(added_rows) = newest_snapshot_added_rows {
@@ -938,7 +999,7 @@ impl ExecutionPlan for IcebergCommitExec {
                 table_meta
                     .metadata_log
                     .push(crate::spec::metadata::table_metadata::MetadataLog {
-                        timestamp_ms,
+                        timestamp_ms: previous_metadata_timestamp_ms,
                         metadata_file: catalog_metadata_location
                             .clone()
                             .unwrap_or_else(|| latest_meta.clone()),
@@ -965,9 +1026,9 @@ impl ExecutionPlan for IcebergCommitExec {
 
                 log::trace!(
                     "Writing metadata: {} snapshot_id={:?} table_url={}",
-                    &new_meta_rel,
+                    new_meta_rel,
                     table_meta.current_snapshot_id,
-                    &table_url
+                    table_url
                 );
 
                 let new_meta_path = object_store::path::Path::from(new_meta_rel.as_str());
@@ -1056,9 +1117,7 @@ impl ExecutionPlan for IcebergCommitExec {
                     .await?;
                 }
 
-                let array = Arc::new(UInt64Array::from(vec![commit_info.row_count]));
-                let batch = RecordBatch::try_new(schema, vec![array])?;
-                return Ok(batch);
+                return commit_count_batch(schema, commit_info.row_count);
             }
         };
 
@@ -1089,3 +1148,92 @@ fn commit_conflict_error() -> DataFusionError {
         "Iceberg commit failed after {MAX_COMMIT_RETRIES} retries due to concurrent metadata updates"
     ))
 }
+
+#[cfg(test)]
+#[expect(clippy::expect_used)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::{Arc, Barrier, Mutex};
+
+    use super::*;
+    use crate::spec::FormatVersion;
+
+    fn table_metadata_at_snapshot(snapshot_id: Option) -> TableMetadata {
+        TableMetadata {
+            format_version: FormatVersion::V2,
+            table_uuid: None,
+            location: "file:///tmp/table".to_string(),
+            last_sequence_number: 2,
+            last_updated_ms: 0,
+            last_column_id: 0,
+            schemas: vec![],
+            current_schema_id: 0,
+            partition_specs: vec![],
+            default_spec_id: 0,
+            last_partition_id: 0,
+            properties: HashMap::new(),
+            current_snapshot_id: snapshot_id,
+            next_row_id: None,
+            encryption_keys: vec![],
+            snapshots: vec![],
+            snapshot_log: vec![],
+            metadata_log: vec![],
+            sort_orders: vec![],
+            default_sort_order_id: None,
+            refs: HashMap::new(),
+            statistics: vec![],
+            partition_statistics: vec![],
+        }
+    }
+
+    #[test]
+    fn planned_delete_snapshot_requirement_rejects_concurrent_branch_advance() {
+        let metadata = Arc::new(Mutex::new(table_metadata_at_snapshot(Some(1))));
+        let barrier = Arc::new(Barrier::new(2));
+        let delete_metadata = Arc::clone(&metadata);
+        let delete_barrier = Arc::clone(&barrier);
+        let delete = std::thread::spawn(move || {
+            let requirement = {
+                let metadata = delete_metadata.lock().expect("metadata lock");
+                expected_snapshot_requirement(Some(metadata.current_snapshot_id))
+                    .expect("DELETE must capture its read snapshot")
+            };
+            delete_barrier.wait();
+            delete_barrier.wait();
+            let metadata = delete_metadata.lock().expect("metadata lock");
+            IcebergCommitExec::validate_requirements(Some(&metadata), &[requirement])
+        });
+
+        barrier.wait();
+        metadata.lock().expect("metadata lock").current_snapshot_id = Some(2);
+        barrier.wait();
+
+        let error = delete
+            .join()
+            .expect("DELETE validation thread")
+            .expect_err("planned snapshot 1 must conflict with current snapshot 2");
+
+        assert!(error.to_string().contains("expected snapshot Some(1)"));
+        assert!(error.to_string().contains("found Some(2)"));
+    }
+
+    #[test]
+    fn empty_read_snapshot_requirement_preserves_none() {
+        let requirement = expected_snapshot_requirement(Some(None))
+            .expect("planned empty snapshot must produce a requirement");
+        assert!(
+            IcebergCommitExec::validate_requirements(
+                Some(&table_metadata_at_snapshot(None)),
+                std::slice::from_ref(&requirement),
+            )
+            .is_ok()
+        );
+        assert!(
+            IcebergCommitExec::validate_requirements(
+                Some(&table_metadata_at_snapshot(Some(2))),
+                &[requirement],
+            )
+            .is_err()
+        );
+    }
+}
diff --git a/crates/sail-iceberg/src/physical_plan/commit/mod.rs b/crates/sail-iceberg/src/physical_plan/commit/mod.rs
index babcd3140..dc863a8dc 100644
--- a/crates/sail-iceberg/src/physical_plan/commit/mod.rs
+++ b/crates/sail-iceberg/src/physical_plan/commit/mod.rs
@@ -15,20 +15,22 @@ pub mod commit_exec;
 use sail_common_datafusion::catalog::LakehouseExecutionContext;
 use serde::{Deserialize, Serialize};
 
-use crate::spec::{DataFile, Operation, PartitionSpec, Schema, TableRequirement, TableUpdate};
+use crate::operations::SnapshotUpdateKind;
+use crate::spec::{DataFile, PartitionSpec, Schema, TableRequirement, TableUpdate};
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct IcebergCommitInfo {
     pub table_uri: String,
     pub row_count: u64,
     pub data_files: Vec,
+    pub delete_files: Vec,
     pub manifest_path: String,
     pub manifest_list_path: String,
     pub updates: Vec,
     pub requirements: Vec,
     pub table_properties: Vec<(String, String)>,
     pub lakehouse_table: Option,
-    pub operation: Operation,
+    pub snapshot_update_kind: SnapshotUpdateKind,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub schema: Option,
     #[serde(skip_serializing_if = "Option::is_none")]
diff --git a/crates/sail-iceberg/src/physical_plan/delete_apply_exec.rs b/crates/sail-iceberg/src/physical_plan/delete_apply_exec.rs
index 51e543a31..eebe3f0d9 100644
--- a/crates/sail-iceberg/src/physical_plan/delete_apply_exec.rs
+++ b/crates/sail-iceberg/src/physical_plan/delete_apply_exec.rs
@@ -13,7 +13,7 @@ use datafusion::physical_expr::EquivalenceProperties;
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
+    DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties,
     SendableRecordBatchStream,
 };
 use datafusion_common::{DataFusionError, Result};
@@ -23,8 +23,8 @@ use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStream
 use url::Url;
 
 use crate::io::StoreContext;
-use crate::spec::delete_index::DeleteFileRef;
 use crate::spec::Schema as IcebergSchema;
+use crate::spec::delete_index::DeleteFileRef;
 
 /// Column name used in Iceberg position-delete files for the target data-file path.
 const POS_DELETE_FILE_PATH_COL: &str = "file_path";
@@ -142,6 +142,18 @@ impl ExecutionPlan for IcebergDeleteApplyExec {
         vec![&self.input]
     }
 
+    fn required_input_distribution(&self) -> Vec {
+        vec![Distribution::SinglePartition]
+    }
+
+    fn benefits_from_input_partitioning(&self) -> Vec {
+        vec![false]
+    }
+
+    fn maintains_input_order(&self) -> Vec {
+        vec![true]
+    }
+
     fn with_new_children(
         self: Arc,
         children: Vec>,
@@ -516,7 +528,7 @@ mod tests {
     #[test]
     fn mask_respects_row_offset_window() {
         let batch = make_batch(); // 5 rows
-                                  // Upstream row offset 10 means this batch spans rows [10, 15).
+        // Upstream row offset 10 means this batch spans rows [10, 15).
         let positions = vec![9u64, 11, 14, 20];
         let mask = compute_delete_mask(&batch, 10, &positions, &[]).unwrap();
         // 9 < 10 → out; 11 → drop idx 1; 14 → drop idx 4; 20 → out.
diff --git a/crates/sail-iceberg/src/physical_plan/delete_writer_common.rs b/crates/sail-iceberg/src/physical_plan/delete_writer_common.rs
new file mode 100644
index 000000000..4448897d8
--- /dev/null
+++ b/crates/sail-iceberg/src/physical_plan/delete_writer_common.rs
@@ -0,0 +1,125 @@
+use datafusion::execution::context::TaskContext;
+use datafusion_common::{DataFusionError, Result};
+use object_store::ObjectStoreExt;
+use object_store::path::Path as ObjectPath;
+use url::Url;
+
+use crate::io::StoreContext;
+use crate::operations::write::arrow_parquet::ArrowParquetWriter;
+use crate::operations::write::base_writer::DataFileWriter;
+use crate::physical_plan::write_location;
+use crate::spec::types::values::Literal;
+use crate::spec::{DataFile, TableMetadata};
+use crate::table::metadata_loader::{
+    load_metadata_file_bytes, metadata_location_to_object_path_string,
+};
+use crate::table_format::{
+    catalog_managed_iceberg_from_properties, metadata_location_from_properties,
+};
+
+pub(crate) fn store_context(context: &TaskContext, location_url: &Url) -> Result {
+    let object_store = context
+        .runtime_env()
+        .object_store_registry
+        .get_store(location_url)
+        .map_err(|e| DataFusionError::External(Box::new(e)))?;
+    StoreContext::new(object_store, location_url)
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct IcebergDeleteWriterConfig {
+    table_url: Url,
+    table_properties: Vec<(String, String)>,
+    write_data_path: Option,
+    write_folder_storage_path: Option,
+}
+
+impl IcebergDeleteWriterConfig {
+    pub(crate) fn new(
+        table_url: Url,
+        table_properties: Vec<(String, String)>,
+        write_data_path: Option,
+        write_folder_storage_path: Option,
+    ) -> Self {
+        Self {
+            table_url,
+            table_properties,
+            write_data_path,
+            write_folder_storage_path,
+        }
+    }
+
+    pub(crate) fn table_url(&self) -> &Url {
+        &self.table_url
+    }
+
+    pub(crate) fn table_properties(&self) -> &[(String, String)] {
+        &self.table_properties
+    }
+
+    pub(crate) fn write_data_path(&self) -> Option<&str> {
+        self.write_data_path.as_deref()
+    }
+
+    pub(crate) fn write_folder_storage_path(&self) -> Option<&str> {
+        self.write_folder_storage_path.as_deref()
+    }
+
+    pub(crate) async fn load_current_table_metadata(
+        &self,
+        store_ctx: &StoreContext,
+    ) -> Result {
+        load_current_table_metadata(store_ctx, &self.table_url, &self.table_properties).await
+    }
+
+    pub(crate) fn resolve_data_location(&self, table_meta: &TableMetadata) -> Result {
+        write_location::resolve_data_location_from_options_and_properties(
+            self.write_data_path(),
+            self.write_folder_storage_path(),
+            &table_meta.properties,
+            &self.table_url,
+        )
+    }
+}
+
+pub(crate) async fn load_current_table_metadata(
+    store_ctx: &StoreContext,
+    table_url: &Url,
+    table_properties: &[(String, String)],
+) -> Result {
+    let metadata_file = if catalog_managed_iceberg_from_properties(table_properties) {
+        match metadata_location_from_properties(table_properties) {
+            Some(location) => metadata_location_to_object_path_string(&location)?,
+            None => crate::table::find_latest_metadata_file(&store_ctx.base, table_url).await?,
+        }
+    } else {
+        crate::table::find_latest_metadata_file(&store_ctx.base, table_url).await?
+    };
+    let bytes = load_metadata_file_bytes(&store_ctx.base, &metadata_file).await?;
+    TableMetadata::from_json(&bytes).map_err(|e| DataFusionError::External(Box::new(e)))
+}
+
+pub(crate) async fn write_delete_parquet_file(
+    data_store_ctx: &StoreContext,
+    data_url: &Url,
+    file_prefix: &str,
+    writer: ArrowParquetWriter,
+    partition_spec_id: i32,
+    partition: Vec>,
+) -> Result {
+    let (bytes, meta) = writer.close().await.map_err(DataFusionError::Execution)?;
+
+    let relative_path = write_location::parquet_file_name(file_prefix);
+    let path = ObjectPath::from(relative_path.as_str());
+    data_store_ctx
+        .prefixed
+        .put(&path, object_store::PutPayload::from(bytes))
+        .await
+        .map_err(|e| DataFusionError::External(Box::new(e)))?;
+    let delete_file_path = write_location::manifest_file_path(data_url, &relative_path);
+
+    DataFileWriter::new(partition_spec_id, delete_file_path, partition)
+        .finish(meta)
+        .map(|outcome| outcome.data_file)
+        .map_err(DataFusionError::Execution)
+}
diff --git a/crates/sail-iceberg/src/physical_plan/discovery_exec.rs b/crates/sail-iceberg/src/physical_plan/discovery_exec.rs
index f807bb7e2..12220ef1e 100644
--- a/crates/sail-iceberg/src/physical_plan/discovery_exec.rs
+++ b/crates/sail-iceberg/src/physical_plan/discovery_exec.rs
@@ -13,7 +13,7 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
     PlanProperties, SendableRecordBatchStream,
 };
-use datafusion_common::{internal_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, internal_err};
 use futures::TryStreamExt;
 
 pub const COL_PARTITION_SCAN: &str = "partition_scan";
@@ -216,9 +216,11 @@ mod tests {
         assert!(schema.is_some());
         // upstream (2 fields) + partition_scan (1 field) = 3
         assert_eq!(schema.as_ref().map(|s| s.fields().len()), Some(3));
-        assert!(schema
-            .as_ref()
-            .map(|s| s.field_with_name(COL_PARTITION_SCAN).is_ok())
-            .is_some_and(|v| v));
+        assert!(
+            schema
+                .as_ref()
+                .map(|s| s.field_with_name(COL_PARTITION_SCAN).is_ok())
+                .is_some_and(|v| v)
+        );
     }
 }
diff --git a/crates/sail-iceberg/src/physical_plan/equality_delete_writer_exec.rs b/crates/sail-iceberg/src/physical_plan/equality_delete_writer_exec.rs
new file mode 100644
index 000000000..94ff7cc58
--- /dev/null
+++ b/crates/sail-iceberg/src/physical_plan/equality_delete_writer_exec.rs
@@ -0,0 +1,450 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use datafusion::arrow::array::ArrayRef;
+use datafusion::arrow::compute::concat_batches;
+use datafusion::arrow::datatypes::{FieldRef, Schema, SchemaRef};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::execution::context::TaskContext;
+use datafusion::physical_expr::{Distribution, EquivalenceProperties};
+use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
+    PlanProperties, SendableRecordBatchStream,
+};
+use datafusion_common::{DataFusionError, Result, internal_err};
+use futures::StreamExt;
+use futures::stream::once;
+use parquet::file::properties::WriterProperties;
+use url::Url;
+
+use crate::datasource::type_converter::iceberg_field_to_arrow;
+use crate::operations::write::arrow_parquet::ArrowParquetWriter;
+use crate::physical_plan::action_schema::{
+    CommitMeta, encode_commit_meta, encode_delete_data_files, iceberg_action_schema,
+};
+use crate::physical_plan::delete_writer_common::{self, IcebergDeleteWriterConfig};
+use crate::spec::types::{PrimitiveType, Type};
+use crate::spec::{
+    DataContentType, DataFile, FormatVersion, MAIN_BRANCH, TableMetadata, TableRequirement,
+};
+
+#[derive(Debug, Clone)]
+pub struct IcebergEqualityDeleteWriterExec {
+    input: Arc,
+    writer_config: IcebergDeleteWriterConfig,
+    lakehouse_table: Option,
+    cache: Arc,
+}
+
+impl IcebergEqualityDeleteWriterExec {
+    pub fn new(
+        input: Arc,
+        table_url: Url,
+        table_properties: Vec<(String, String)>,
+        write_data_path: Option,
+        write_folder_storage_path: Option,
+        lakehouse_table: Option,
+    ) -> Self {
+        let schema = iceberg_action_schema().unwrap_or_else(|e| {
+            log::error!("failed to initialize iceberg action schema: {e}");
+            Arc::new(Schema::empty())
+        });
+        let cache = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(schema),
+            Partitioning::UnknownPartitioning(1),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        Self {
+            input,
+            writer_config: IcebergDeleteWriterConfig::new(
+                table_url,
+                table_properties,
+                write_data_path,
+                write_folder_storage_path,
+            ),
+            lakehouse_table,
+            cache,
+        }
+    }
+
+    pub fn input(&self) -> &Arc {
+        &self.input
+    }
+
+    pub fn table_url(&self) -> &Url {
+        self.writer_config.table_url()
+    }
+
+    pub fn table_properties(&self) -> &[(String, String)] {
+        self.writer_config.table_properties()
+    }
+
+    pub fn write_data_path(&self) -> Option<&str> {
+        self.writer_config.write_data_path()
+    }
+
+    pub fn write_folder_storage_path(&self) -> Option<&str> {
+        self.writer_config.write_folder_storage_path()
+    }
+
+    pub fn lakehouse_table(
+        &self,
+    ) -> Option<&sail_common_datafusion::catalog::LakehouseExecutionContext> {
+        self.lakehouse_table.as_ref()
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for IcebergEqualityDeleteWriterExec {
+    fn name(&self) -> &str {
+        "IcebergEqualityDeleteWriterExec"
+    }
+
+    fn properties(&self) -> &Arc {
+        &self.cache
+    }
+
+    fn required_input_distribution(&self) -> Vec {
+        // FIXME: Write equality deletes in parallel and roll files by target size once
+        // writer commit metadata can be merged safely across output partitions.
+        vec![Distribution::SinglePartition]
+    }
+
+    fn children(&self) -> Vec<&Arc> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc,
+        children: Vec>,
+    ) -> Result> {
+        if children.len() != 1 {
+            return internal_err!("IcebergEqualityDeleteWriterExec requires exactly one child");
+        }
+        Ok(Arc::new(Self::new(
+            Arc::clone(&children[0]),
+            self.writer_config.table_url().clone(),
+            self.writer_config.table_properties().to_vec(),
+            self.writer_config
+                .write_data_path()
+                .map(ToString::to_string),
+            self.writer_config
+                .write_folder_storage_path()
+                .map(ToString::to_string),
+            self.lakehouse_table.clone(),
+        )))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc,
+    ) -> Result {
+        if partition != 0 {
+            return internal_err!(
+                "IcebergEqualityDeleteWriterExec can only be executed in a single partition"
+            );
+        }
+        let input_partitions = self.input.output_partitioning().partition_count();
+        if input_partitions != 1 {
+            return internal_err!(
+                "IcebergEqualityDeleteWriterExec requires exactly one input partition, got {input_partitions}"
+            );
+        }
+
+        let input = self.input.execute(0, Arc::clone(&context))?;
+        let input_schema = self.input.schema();
+        let writer_config = self.writer_config.clone();
+        let lakehouse_table = self.lakehouse_table.clone();
+        let output_schema = self.schema();
+        let schema_for_adapter = output_schema.clone();
+
+        let future = async move {
+            let table_store_ctx =
+                delete_writer_common::store_context(&context, writer_config.table_url())?;
+            let table_meta = writer_config
+                .load_current_table_metadata(&table_store_ctx)
+                .await?;
+            if table_meta.format_version < FormatVersion::V2 {
+                return Err(DataFusionError::Plan(
+                    "Iceberg equality delete writes require table format-version 2 or higher"
+                        .to_string(),
+                ));
+            }
+            let current_schema = table_meta.current_schema().ok_or_else(|| {
+                DataFusionError::Plan(
+                    "Iceberg table metadata is missing current schema".to_string(),
+                )
+            })?;
+            let default_spec = table_meta
+                .default_partition_spec()
+                .cloned()
+                .unwrap_or_else(crate::spec::PartitionSpec::unpartitioned_spec);
+            if !default_spec.is_unpartitioned() {
+                return Err(DataFusionError::NotImplemented(
+                    "Iceberg equality delete writes for partitioned tables are not supported yet"
+                        .to_string(),
+                ));
+            }
+            let data_location = writer_config.resolve_data_location(&table_meta)?;
+            let data_store_ctx = delete_writer_common::store_context(&context, &data_location)?;
+
+            // TODO: Prefer identifier/configured equality fields over full-row keys.
+            let delete_spec = EqualityDeleteSpec::full_row(current_schema, &input_schema)?;
+
+            let mut stream = input;
+            let mut writer: Option = None;
+            let mut total_rows = 0u64;
+            while let Some(batch_result) = stream.next().await {
+                let batch = batch_result?;
+                if batch.num_rows() == 0 {
+                    continue;
+                }
+                let delete_batch = delete_spec.project_batch(&batch)?;
+                let row_count = delete_batch.num_rows();
+                total_rows = total_rows
+                    .checked_add(u64::try_from(row_count).map_err(|e| {
+                        DataFusionError::Execution(format!("Row count overflow: {e}"))
+                    })?)
+                    .ok_or_else(|| DataFusionError::Execution("Row count overflow".to_string()))?;
+
+                let writer = match writer.as_mut() {
+                    Some(writer) => writer,
+                    None => writer.insert(
+                        ArrowParquetWriter::try_new(
+                            delete_spec.arrow_schema.as_ref(),
+                            WriterProperties::default(),
+                        )
+                        .map_err(DataFusionError::Execution)?,
+                    ),
+                };
+                writer
+                    .write_batch(&delete_batch)
+                    .await
+                    .map_err(DataFusionError::Execution)?;
+            }
+
+            let Some(writer) = writer else {
+                return encode_delete_data_files(vec![]);
+            };
+
+            let delete_file = write_equality_delete_file(
+                &data_store_ctx,
+                &data_location,
+                writer,
+                default_spec.spec_id(),
+                total_rows,
+                delete_spec.equality_ids.clone(),
+            )
+            .await?;
+            let requirements = commit_requirements(&table_meta);
+            let commit_meta = CommitMeta {
+                table_uri: writer_config.table_url().to_string(),
+                row_count: total_rows,
+                requirements,
+                table_properties: writer_config.table_properties().to_vec(),
+                lakehouse_table,
+                schema: None,
+                partition_spec: None,
+            };
+
+            let schema = iceberg_action_schema()?;
+            let batches = vec![
+                encode_delete_data_files(vec![delete_file])?,
+                encode_commit_meta(commit_meta)?,
+            ];
+            concat_batches(&schema, &batches)
+                .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+        };
+
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            schema_for_adapter,
+            once(future),
+        )))
+    }
+}
+
+impl DisplayAs for IcebergEqualityDeleteWriterExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default
+            | DisplayFormatType::Verbose
+            | DisplayFormatType::TreeRender => {
+                write!(
+                    f,
+                    "IcebergEqualityDeleteWriterExec(table_path={})",
+                    self.writer_config.table_url()
+                )
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct EqualityDeleteField {
+    field_id: i32,
+    name: String,
+    arrow_field: FieldRef,
+}
+
+#[derive(Debug, Clone)]
+struct EqualityDeleteSpec {
+    fields: Vec,
+    arrow_schema: SchemaRef,
+    equality_ids: Vec,
+}
+
+impl EqualityDeleteSpec {
+    fn full_row(iceberg_schema: &crate::spec::Schema, input_schema: &SchemaRef) -> Result {
+        let fields = equality_delete_fields(iceberg_schema, input_schema)?;
+        let arrow_schema = Arc::new(Schema::new(
+            fields
+                .iter()
+                .map(|field| field.arrow_field.clone())
+                .collect::>(),
+        ));
+        let equality_ids = fields.iter().map(|field| field.field_id).collect();
+        Ok(Self {
+            fields,
+            arrow_schema,
+            equality_ids,
+        })
+    }
+
+    fn project_batch(&self, batch: &RecordBatch) -> Result {
+        project_delete_batch(batch, &self.fields, self.arrow_schema.clone())
+    }
+}
+
+fn equality_delete_fields(
+    iceberg_schema: &crate::spec::Schema,
+    input_schema: &SchemaRef,
+) -> Result> {
+    let mut fields = Vec::with_capacity(iceberg_schema.fields().len());
+    for field in iceberg_schema.fields() {
+        validate_equality_delete_type(&field.name, &field.field_type)?;
+        let arrow_field = Arc::new(iceberg_field_to_arrow(field)?);
+        let input_field = input_schema.field_with_name(&field.name).map_err(|_| {
+            DataFusionError::Plan(format!(
+                "Iceberg equality delete input is missing column '{}'",
+                field.name
+            ))
+        })?;
+        if input_field.data_type() != arrow_field.data_type() {
+            return Err(DataFusionError::Plan(format!(
+                "Iceberg equality delete column '{}' has input type {:?}, expected {:?}",
+                field.name,
+                input_field.data_type(),
+                arrow_field.data_type()
+            )));
+        }
+        fields.push(EqualityDeleteField {
+            field_id: field.id,
+            name: field.name.clone(),
+            arrow_field,
+        });
+    }
+    Ok(fields)
+}
+
+fn validate_equality_delete_type(name: &str, ty: &Type) -> Result<()> {
+    match ty {
+        Type::Primitive(PrimitiveType::Float | PrimitiveType::Double) => {
+            Err(DataFusionError::Plan(format!(
+                "Iceberg equality delete column '{name}' has identifier-field-invalid type {ty}; float and double cannot be equality delete keys"
+            )))
+        }
+        // TODO: Add equality writer support for variant/geometry/geography encodings.
+        Type::Primitive(
+            PrimitiveType::Unknown
+            | PrimitiveType::Variant
+            | PrimitiveType::Geometry { .. }
+            | PrimitiveType::Geography { .. },
+        ) => Err(DataFusionError::NotImplemented(format!(
+            "Iceberg equality delete writes for column '{name}' with type {ty} are not supported yet"
+        ))),
+        Type::Primitive(_) => Ok(()),
+        // TODO: Support nested equality fields once projection preserves nested field IDs.
+        Type::Struct(_) | Type::List(_) | Type::Map(_) => {
+            Err(DataFusionError::NotImplemented(format!(
+                "Iceberg equality delete writes for nested column '{name}' with type {ty} are not supported yet"
+            )))
+        }
+    }
+}
+
+fn project_delete_batch(
+    batch: &RecordBatch,
+    delete_fields: &[EqualityDeleteField],
+    delete_schema: SchemaRef,
+) -> Result {
+    let mut columns: Vec = Vec::with_capacity(delete_fields.len());
+    for field in delete_fields {
+        let col = batch.column_by_name(&field.name).ok_or_else(|| {
+            DataFusionError::Internal(format!(
+                "Iceberg equality delete batch is missing column '{}'",
+                field.name
+            ))
+        })?;
+        if col.data_type() != field.arrow_field.data_type() {
+            return Err(DataFusionError::Internal(format!(
+                "Iceberg equality delete column '{}' has batch type {:?}, expected {:?}",
+                field.name,
+                col.data_type(),
+                field.arrow_field.data_type()
+            )));
+        }
+        columns.push(col.clone());
+    }
+    RecordBatch::try_new(delete_schema, columns)
+        .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+}
+
+async fn write_equality_delete_file(
+    data_store_ctx: &crate::io::StoreContext,
+    data_url: &Url,
+    writer: ArrowParquetWriter,
+    partition_spec_id: i32,
+    total_rows: u64,
+    equality_ids: Vec,
+) -> Result {
+    let mut delete_file = delete_writer_common::write_delete_parquet_file(
+        data_store_ctx,
+        data_url,
+        "equality-delete",
+        writer,
+        partition_spec_id,
+        Vec::new(),
+    )
+    .await?;
+    delete_file.content = DataContentType::EqualityDeletes;
+    delete_file.record_count = total_rows;
+    delete_file.referenced_data_file = None;
+    delete_file.sort_order_id = None;
+    delete_file.equality_ids = equality_ids;
+    Ok(delete_file)
+}
+
+fn commit_requirements(table_meta: &TableMetadata) -> Vec {
+    vec![
+        TableRequirement::LastAssignedFieldIdMatch {
+            last_assigned_field_id: table_meta.last_column_id,
+        },
+        TableRequirement::CurrentSchemaIdMatch {
+            current_schema_id: table_meta.current_schema_id,
+        },
+        TableRequirement::LastAssignedPartitionIdMatch {
+            last_assigned_partition_id: table_meta.last_partition_id,
+        },
+        TableRequirement::DefaultSpecIdMatch {
+            default_spec_id: table_meta.default_spec_id,
+        },
+        TableRequirement::RefSnapshotIdMatch {
+            r#ref: MAIN_BRANCH.to_string(),
+            snapshot_id: table_meta.current_snapshot_id.filter(|id| *id >= 0),
+        },
+    ]
+}
diff --git a/crates/sail-iceberg/src/physical_plan/manifest_scan_exec.rs b/crates/sail-iceberg/src/physical_plan/manifest_scan_exec.rs
index 6c4e023db..5c0a616ab 100644
--- a/crates/sail-iceberg/src/physical_plan/manifest_scan_exec.rs
+++ b/crates/sail-iceberg/src/physical_plan/manifest_scan_exec.rs
@@ -16,7 +16,7 @@ use futures::stream::{self, TryStreamExt};
 use url::Url;
 
 use crate::io::{
-    load_manifest as io_load_manifest, load_manifest_list as io_load_manifest_list, StoreContext,
+    StoreContext, load_manifest as io_load_manifest, load_manifest_list as io_load_manifest_list,
 };
 use crate::spec::{ManifestContentType, ManifestFile, ManifestStatus, Snapshot};
 
diff --git a/crates/sail-iceberg/src/physical_plan/merge_metadata_exec.rs b/crates/sail-iceberg/src/physical_plan/merge_metadata_exec.rs
new file mode 100644
index 000000000..841b697ac
--- /dev/null
+++ b/crates/sail-iceberg/src/physical_plan/merge_metadata_exec.rs
@@ -0,0 +1,308 @@
+use std::sync::Arc;
+
+use async_stream::try_stream;
+use async_trait::async_trait;
+use datafusion::arrow::array::{Array, ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray};
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::execution::context::TaskContext;
+use datafusion::physical_expr::EquivalenceProperties;
+use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
+    Partitioning, PlanProperties, SendableRecordBatchStream,
+};
+use datafusion_common::{DataFusionError, Result};
+use futures::stream::TryStreamExt;
+
+use crate::row_level_metadata::{
+    MERGE_PARTITION_COLUMN, MERGE_PARTITION_SPEC_ID_COLUMN, RowLevelMetadataColumns,
+};
+
+#[derive(Debug, Clone)]
+pub struct IcebergMergeMetadataExec {
+    input: Arc,
+    data_file_path: Option,
+    data_file_partition_spec_id: Option,
+    data_file_partition_json: Option,
+    file_column_name: Option,
+    row_index_column_name: Option,
+    output_schema: SchemaRef,
+    cache: Arc,
+}
+
+impl IcebergMergeMetadataExec {
+    pub fn try_new(
+        input: Arc,
+        data_file_path: String,
+        data_file_partition_spec_id: i32,
+        data_file_partition_json: String,
+        file_column_name: Option,
+        row_index_column_name: Option,
+    ) -> Result {
+        Self::try_new_with_path_source(
+            input,
+            Some(data_file_path),
+            Some(data_file_partition_spec_id),
+            Some(data_file_partition_json),
+            file_column_name,
+            row_index_column_name,
+        )
+    }
+
+    pub fn try_new_partitioned_files(
+        input: Arc,
+        file_column_name: String,
+        row_index_column_name: Option,
+    ) -> Result {
+        for metadata_column in [
+            file_column_name.as_str(),
+            MERGE_PARTITION_SPEC_ID_COLUMN,
+            MERGE_PARTITION_COLUMN,
+        ] {
+            if input.schema().field_with_name(metadata_column).is_err() {
+                return Err(DataFusionError::Plan(format!(
+                    "Iceberg merge scan is missing metadata column '{metadata_column}'"
+                )));
+            }
+        }
+        Self::try_new_with_path_source(
+            input,
+            None,
+            None,
+            None,
+            Some(file_column_name),
+            row_index_column_name,
+        )
+    }
+
+    fn try_new_with_path_source(
+        input: Arc,
+        data_file_path: Option,
+        data_file_partition_spec_id: Option,
+        data_file_partition_json: Option,
+        file_column_name: Option,
+        row_index_column_name: Option,
+    ) -> Result {
+        let appended_file_column = data_file_path
+            .is_some()
+            .then_some(file_column_name.as_deref())
+            .flatten();
+        let metadata_columns =
+            RowLevelMetadataColumns::new(appended_file_column, row_index_column_name.as_deref());
+        let metadata_columns = if data_file_path.is_some() {
+            metadata_columns.with_delete_file_metadata()
+        } else {
+            metadata_columns
+        };
+        let output_schema = Arc::new(metadata_columns.append_to_schema(input.schema().as_ref())?);
+        let partition_count = input.output_partitioning().partition_count().max(1);
+        let cache = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(output_schema.clone()),
+            Partitioning::UnknownPartitioning(partition_count),
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ));
+        Ok(Self {
+            input,
+            data_file_path,
+            data_file_partition_spec_id,
+            data_file_partition_json,
+            file_column_name,
+            row_index_column_name,
+            output_schema,
+            cache,
+        })
+    }
+
+    pub fn input(&self) -> &Arc {
+        &self.input
+    }
+
+    pub fn data_file_path(&self) -> Option<&str> {
+        self.data_file_path.as_deref()
+    }
+
+    pub fn file_column_name(&self) -> Option<&str> {
+        self.file_column_name.as_deref()
+    }
+
+    pub fn data_file_partition_spec_id(&self) -> Option {
+        self.data_file_partition_spec_id
+    }
+
+    pub fn data_file_partition_json(&self) -> Option<&str> {
+        self.data_file_partition_json.as_deref()
+    }
+
+    pub fn row_index_column_name(&self) -> Option<&str> {
+        self.row_index_column_name.as_deref()
+    }
+}
+
+#[async_trait]
+impl ExecutionPlan for IcebergMergeMetadataExec {
+    fn name(&self) -> &str {
+        "IcebergMergeMetadataExec"
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.output_schema.clone()
+    }
+
+    fn required_input_distribution(&self) -> Vec {
+        if self.data_file_path.is_some() {
+            vec![Distribution::SinglePartition]
+        } else {
+            vec![Distribution::UnspecifiedDistribution]
+        }
+    }
+
+    fn benefits_from_input_partitioning(&self) -> Vec {
+        vec![false]
+    }
+
+    fn maintains_input_order(&self) -> Vec {
+        vec![true]
+    }
+
+    fn children(&self) -> Vec<&Arc> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc,
+        children: Vec>,
+    ) -> Result> {
+        if children.len() != 1 {
+            return Err(DataFusionError::Internal(
+                "IcebergMergeMetadataExec requires exactly one child".to_string(),
+            ));
+        }
+        Ok(Arc::new(Self::try_new_with_path_source(
+            Arc::clone(&children[0]),
+            self.data_file_path.clone(),
+            self.data_file_partition_spec_id,
+            self.data_file_partition_json.clone(),
+            self.file_column_name.clone(),
+            self.row_index_column_name.clone(),
+        )?))
+    }
+
+    fn properties(&self) -> &Arc {
+        &self.cache
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc,
+    ) -> Result {
+        if self.data_file_path.is_some() && partition != 0 {
+            return Err(DataFusionError::Internal(format!(
+                "IcebergMergeMetadataExec only supports partition 0, got {partition}"
+            )));
+        }
+
+        let child = self.input.execute(partition, context)?;
+        let output_schema = self.output_schema.clone();
+        let schema_for_adapter = output_schema.clone();
+        let data_file_path = self.data_file_path.clone();
+        let data_file_partition_spec_id = self.data_file_partition_spec_id;
+        let data_file_partition_json = self.data_file_partition_json.clone();
+        let file_column_name = self.file_column_name.clone();
+        let include_file = data_file_path.is_some() && file_column_name.is_some();
+        let include_row_index = self.row_index_column_name.is_some();
+
+        let stream = try_stream! {
+            // The provider keeps complete, naturally ordered files in each input
+            // partition, so this offset is file-absolute.
+            let mut row_offset = 0i64;
+            let mut current_file_path: Option = None;
+            let mut stream = child;
+            while let Some(batch) = stream.try_next().await? {
+                let rows = batch.num_rows();
+                let mut columns = batch.columns().to_vec();
+                if include_file {
+                    let values = (0..rows)
+                        .map(|_| data_file_path.as_deref())
+                        .collect::>();
+                    columns.push(Arc::new(StringArray::from(values)) as ArrayRef);
+                    columns.push(Arc::new(Int32Array::from(vec![
+                        data_file_partition_spec_id;
+                        rows
+                    ])) as ArrayRef);
+                    columns.push(Arc::new(StringArray::from(vec![
+                        data_file_partition_json.as_deref();
+                        rows
+                    ])) as ArrayRef);
+                }
+                if include_row_index {
+                    let values = if let Some(file_column_name) = file_column_name.as_deref()
+                        && data_file_path.is_none()
+                    {
+                        let file_paths = batch
+                            .column_by_name(file_column_name)
+                            .ok_or_else(|| DataFusionError::Internal(format!(
+                                "missing Iceberg merge file column '{file_column_name}'"
+                            )))?
+                            .as_any()
+                            .downcast_ref::()
+                            .ok_or_else(|| DataFusionError::Internal(format!(
+                                "Iceberg merge file column '{file_column_name}' must be Utf8"
+                            )))?;
+                        let mut values = Vec::with_capacity(rows);
+                        for row in 0..rows {
+                            if file_paths.is_null(row) {
+                                Err(DataFusionError::Execution(
+                                    "Iceberg merge file path cannot be null".to_string(),
+                                ))?;
+                            }
+                            let file_path = file_paths.value(row);
+                            if current_file_path.as_deref() != Some(file_path) {
+                                current_file_path = Some(file_path.to_string());
+                                row_offset = 0;
+                            }
+                            values.push(row_offset);
+                            row_offset += 1;
+                        }
+                        values
+                    } else {
+                        let values = (0..rows)
+                            .map(|idx| row_offset + idx as i64)
+                            .collect::>();
+                        row_offset += rows as i64;
+                        values
+                    };
+                    columns.push(Arc::new(Int64Array::from(values)) as ArrayRef);
+                }
+
+                yield RecordBatch::try_new(output_schema.clone(), columns)?;
+            }
+        };
+
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            schema_for_adapter,
+            Box::pin(stream),
+        )))
+    }
+}
+
+impl DisplayAs for IcebergMergeMetadataExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default
+            | DisplayFormatType::Verbose
+            | DisplayFormatType::TreeRender => match self.data_file_path.as_deref() {
+                Some(data_file_path) => {
+                    write!(f, "IcebergMergeMetadataExec: data_file={data_file_path}")
+                }
+                None => write!(
+                    f,
+                    "IcebergMergeMetadataExec: data_file_column={}",
+                    self.file_column_name.as_deref().unwrap_or("")
+                ),
+            },
+        }
+    }
+}
diff --git a/crates/sail-iceberg/src/physical_plan/merge_row_projection.rs b/crates/sail-iceberg/src/physical_plan/merge_row_projection.rs
new file mode 100644
index 000000000..d2249bb95
--- /dev/null
+++ b/crates/sail-iceberg/src/physical_plan/merge_row_projection.rs
@@ -0,0 +1,189 @@
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, BooleanArray, Int32Array, Int64Array};
+use datafusion::arrow::compute::filter_record_batch;
+use datafusion::arrow::datatypes::{Schema, SchemaRef};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion_common::{DataFusionError, Result};
+use sail_common_datafusion::datasource::{
+    MERGE_FILE_COLUMN, MERGE_ROW_INDEX_COLUMN, MERGE_SOURCE_METRIC_COLUMN, OPERATION_COLUMN,
+    RowLevelOperationType,
+};
+
+use crate::row_level_metadata::{MERGE_PARTITION_COLUMN, MERGE_PARTITION_SPEC_ID_COLUMN};
+
+#[derive(Debug, Clone)]
+pub(crate) struct IcebergMergeRowProjection {
+    operation_index: usize,
+    data_indices: Vec,
+    data_schema: SchemaRef,
+}
+
+impl IcebergMergeRowProjection {
+    pub(crate) fn try_new(input_schema: SchemaRef) -> Result {
+        let operation_index = input_schema.index_of(OPERATION_COLUMN)?;
+        let internal_columns = [
+            MERGE_FILE_COLUMN,
+            MERGE_ROW_INDEX_COLUMN,
+            MERGE_SOURCE_METRIC_COLUMN,
+            OPERATION_COLUMN,
+            MERGE_PARTITION_SPEC_ID_COLUMN,
+            MERGE_PARTITION_COLUMN,
+        ];
+        let data_indices = input_schema
+            .fields()
+            .iter()
+            .enumerate()
+            .filter_map(|(index, field)| {
+                (!internal_columns.contains(&field.name().as_str())).then_some(index)
+            })
+            .collect::>();
+        let fields = data_indices
+            .iter()
+            .map(|index| input_schema.field(*index).clone())
+            .collect::>();
+        let data_schema = Arc::new(Schema::new_with_metadata(
+            fields,
+            input_schema.metadata().clone(),
+        ));
+        Ok(Self {
+            operation_index,
+            data_indices,
+            data_schema,
+        })
+    }
+
+    pub(crate) fn data_schema(&self) -> SchemaRef {
+        Arc::clone(&self.data_schema)
+    }
+
+    pub(crate) fn project_data_rows(&self, batch: &RecordBatch) -> Result {
+        let mask = merge_operation_mask(batch, self.operation_index, merge_operation_writes_data)?;
+        let filtered = filter_record_batch(batch, &mask)
+            .map_err(|error| DataFusionError::ArrowError(Box::new(error), None))?;
+        let columns = self
+            .data_indices
+            .iter()
+            .map(|index| filtered.column(*index).clone())
+            .collect::>();
+        Ok(RecordBatch::try_new(
+            Arc::clone(&self.data_schema),
+            columns,
+        )?)
+    }
+
+    pub(crate) fn project_position_delete_rows(&self, batch: &RecordBatch) -> Result {
+        let mask = merge_operation_mask(
+            batch,
+            self.operation_index,
+            merge_operation_writes_position_delete,
+        )?;
+        filter_record_batch(batch, &mask)
+            .map_err(|error| DataFusionError::ArrowError(Box::new(error), None))
+    }
+}
+
+fn merge_operation_mask(
+    batch: &RecordBatch,
+    operation_index: usize,
+    keep: impl Fn(i32) -> bool,
+) -> Result {
+    let column = batch.column(operation_index);
+
+    if let Some(values) = column.as_any().downcast_ref::() {
+        let mask = (0..values.len())
+            .map(|idx| (!values.is_null(idx)).then(|| keep(values.value(idx))))
+            .collect::();
+        return Ok(mask);
+    }
+
+    if let Some(values) = column.as_any().downcast_ref::() {
+        let mask = (0..values.len())
+            .map(|idx| (!values.is_null(idx)).then(|| keep(values.value(idx) as i32)))
+            .collect::();
+        return Ok(mask);
+    }
+
+    Err(DataFusionError::Internal(format!(
+        "{OPERATION_COLUMN} must be Int32 or Int64"
+    )))
+}
+
+fn merge_operation_writes_data(value: i32) -> bool {
+    value == RowLevelOperationType::Insert.as_i32()
+        || value == RowLevelOperationType::MatchedUpdate.as_i32()
+        || value == RowLevelOperationType::NotMatchedBySourceUpdate.as_i32()
+}
+
+fn merge_operation_writes_position_delete(value: i32) -> bool {
+    value == RowLevelOperationType::MatchedDelete.as_i32()
+        || value == RowLevelOperationType::MatchedUpdate.as_i32()
+        || value == RowLevelOperationType::NotMatchedBySourceDelete.as_i32()
+        || value == RowLevelOperationType::NotMatchedBySourceUpdate.as_i32()
+}
+
+#[cfg(test)]
+#[expect(clippy::expect_used)]
+mod tests {
+    use datafusion::arrow::array::{Int32Array, Int64Array, StringArray};
+    use datafusion::arrow::datatypes::{DataType, Field};
+
+    use super::*;
+
+    #[test]
+    fn merge_projection_routes_updates_to_data_and_position_deletes() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, true),
+            Field::new(MERGE_FILE_COLUMN, DataType::Utf8, true),
+            Field::new(MERGE_ROW_INDEX_COLUMN, DataType::Int64, true),
+            Field::new(MERGE_PARTITION_SPEC_ID_COLUMN, DataType::Int32, true),
+            Field::new(MERGE_PARTITION_COLUMN, DataType::Utf8, true),
+            Field::new(OPERATION_COLUMN, DataType::Int32, false),
+            Field::new(MERGE_SOURCE_METRIC_COLUMN, DataType::Int64, true),
+        ]));
+        let batch = RecordBatch::try_new(
+            Arc::clone(&schema),
+            vec![
+                Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3), None])),
+                Arc::new(StringArray::from(vec![
+                    Some("data.parquet"),
+                    Some("data.parquet"),
+                    None,
+                    None,
+                ])),
+                Arc::new(Int64Array::from(vec![Some(0), Some(1), None, None])),
+                Arc::new(Int32Array::from(vec![Some(0), Some(0), None, None])),
+                Arc::new(StringArray::from(vec![Some("[]"), Some("[]"), None, None])),
+                Arc::new(Int32Array::from(vec![
+                    RowLevelOperationType::MatchedUpdate.as_i32(),
+                    RowLevelOperationType::MatchedDelete.as_i32(),
+                    RowLevelOperationType::Insert.as_i32(),
+                    RowLevelOperationType::SourceMetric.as_i32(),
+                ])),
+                Arc::new(Int64Array::from(vec![None, None, None, Some(3)])),
+            ],
+        )
+        .expect("merge batch");
+        let projection = IcebergMergeRowProjection::try_new(schema).expect("merge projection");
+
+        let data_rows = projection.project_data_rows(&batch).expect("data rows");
+        let data_ids = data_rows
+            .column_by_name("id")
+            .expect("id")
+            .as_any()
+            .downcast_ref::()
+            .expect("Int32 ids");
+        assert_eq!(data_ids.values(), &[1, 3]);
+
+        let delete_rows = projection
+            .project_position_delete_rows(&batch)
+            .expect("position-delete rows");
+        let positions = delete_rows
+            .column_by_name(MERGE_ROW_INDEX_COLUMN)
+            .expect("row index")
+            .as_any()
+            .downcast_ref::()
+            .expect("Int64 row indices");
+        assert_eq!(positions.values(), &[0, 1]);
+    }
+}
diff --git a/crates/sail-iceberg/src/physical_plan/mod.rs b/crates/sail-iceberg/src/physical_plan/mod.rs
index eb5abca27..844ee64ed 100644
--- a/crates/sail-iceberg/src/physical_plan/mod.rs
+++ b/crates/sail-iceberg/src/physical_plan/mod.rs
@@ -13,17 +13,25 @@
 pub mod action_schema;
 pub mod commit;
 pub mod delete_apply_exec;
+mod delete_writer_common;
 pub mod discovery_exec;
+pub mod equality_delete_writer_exec;
 pub mod manifest_scan_exec;
+pub mod merge_metadata_exec;
+pub(crate) mod merge_row_projection;
 pub mod plan_builder;
+mod position_delete_writer;
 pub mod scan_by_data_files_exec;
+mod write_location;
 mod writer_exec;
 mod writer_options;
 
 pub use commit::commit_exec::IcebergCommitExec;
 pub use delete_apply_exec::IcebergDeleteApplyExec;
 pub use discovery_exec::IcebergDiscoveryExec;
+pub use equality_delete_writer_exec::IcebergEqualityDeleteWriterExec;
 pub use manifest_scan_exec::IcebergManifestScanExec;
+pub use merge_metadata_exec::IcebergMergeMetadataExec;
 pub use plan_builder::{IcebergPlanBuilder, IcebergTableConfig};
 pub use scan_by_data_files_exec::IcebergScanByDataFilesExec;
 pub use writer_exec::IcebergWriterExec;
diff --git a/crates/sail-iceberg/src/physical_plan/plan_builder.rs b/crates/sail-iceberg/src/physical_plan/plan_builder.rs
index 2fffb7dbf..dcba9720f 100644
--- a/crates/sail-iceberg/src/physical_plan/plan_builder.rs
+++ b/crates/sail-iceberg/src/physical_plan/plan_builder.rs
@@ -24,6 +24,7 @@ use sail_common_datafusion::catalog::CatalogPartitionField;
 use sail_common_datafusion::datasource::PhysicalSinkMode;
 use url::Url;
 
+use crate::operations::SnapshotUpdateKind;
 use crate::physical_plan::writer_exec::IcebergWriterExec;
 use crate::physical_plan::writer_options::IcebergWriterExecOptions;
 use crate::utils::partition_transform::format_partition_expr;
@@ -128,13 +129,14 @@ impl<'a> IcebergPlanBuilder<'a> {
     }
 
     fn add_sort_node(&self, input: Arc) -> Result> {
-        if let Some(sort_exprs) = self.sort_order.clone() {
-            let lex = LexOrdering::new(sort_exprs).ok_or_else(|| {
-                datafusion::common::DataFusionError::Internal("Invalid sort order".to_string())
-            })?;
-            Ok(Arc::new(SortExec::new(lex, input)))
-        } else {
-            Ok(input)
+        match self.sort_order.clone() {
+            Some(sort_exprs) => {
+                let lex = LexOrdering::new(sort_exprs).ok_or_else(|| {
+                    datafusion::common::DataFusionError::Internal("Invalid sort order".to_string())
+                })?;
+                Ok(Arc::new(SortExec::new(lex, input)))
+            }
+            _ => Ok(input),
         }
     }
 
@@ -151,11 +153,19 @@ impl<'a> IcebergPlanBuilder<'a> {
     }
 
     fn add_commit_node(&self, input: Arc) -> Result> {
+        let snapshot_update_kind = if self.table_config.table_exists
+            && matches!(&self.sink_mode, PhysicalSinkMode::Overwrite)
+        {
+            SnapshotUpdateKind::FullOverwrite
+        } else {
+            SnapshotUpdateKind::FastAppend
+        };
         Ok(Arc::new(
             crate::physical_plan::commit::commit_exec::IcebergCommitExec::new(
                 input,
                 self.table_config.table_url.clone(),
                 self.table_config.options.lakehouse_table.clone(),
+                snapshot_update_kind,
             ),
         ))
     }
diff --git a/crates/sail-iceberg/src/physical_plan/position_delete_writer.rs b/crates/sail-iceberg/src/physical_plan/position_delete_writer.rs
new file mode 100644
index 000000000..1d79512f9
--- /dev/null
+++ b/crates/sail-iceberg/src/physical_plan/position_delete_writer.rs
@@ -0,0 +1,254 @@
+use std::collections::btree_map::Entry;
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, Int32Array, Int64Array, StringArray};
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::arrow::record_batch::RecordBatch as ArrowRecordBatch;
+use datafusion_common::{DataFusionError, Result};
+use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
+use parquet::file::properties::WriterProperties;
+use url::Url;
+
+use crate::io::StoreContext;
+use crate::operations::write::arrow_parquet::ArrowParquetWriter;
+use crate::physical_plan::delete_writer_common;
+use crate::row_level_metadata::{MERGE_PARTITION_COLUMN, MERGE_PARTITION_SPEC_ID_COLUMN};
+use crate::spec::types::values::Literal;
+use crate::spec::{DataContentType, DataFile, TableMetadata};
+
+#[derive(Debug, Clone, PartialEq)]
+struct PositionDeleteTarget {
+    file_path: String,
+    partition_spec_id: i32,
+    partition_json: String,
+    partition: Vec>,
+}
+
+#[derive(Debug)]
+struct PositionDeleteRows {
+    target: PositionDeleteTarget,
+    positions: BTreeSet,
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct PositionDeleteAccumulator {
+    // FIXME: Stream sorted positions into rolling delete files and aggregate all
+    // emitted files before commit instead of buffering every position per data file.
+    rows_by_file: BTreeMap,
+}
+
+impl PositionDeleteAccumulator {
+    pub(crate) fn add_batch(
+        &mut self,
+        table_meta: &TableMetadata,
+        batch: &ArrowRecordBatch,
+        file_column_name: &str,
+        row_index_column_name: &str,
+    ) -> Result<()> {
+        let file_paths = batch
+            .column_by_name(file_column_name)
+            .ok_or_else(|| DataFusionError::Internal(format!("missing column {file_column_name}")))?
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!("{file_column_name} must be a Utf8 column"))
+            })?;
+        let row_indices = batch
+            .column_by_name(row_index_column_name)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!("missing column {row_index_column_name}"))
+            })?
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "{row_index_column_name} must be an Int64 column"
+                ))
+            })?;
+        let partition_spec_ids = batch
+            .column_by_name(MERGE_PARTITION_SPEC_ID_COLUMN)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "missing column {MERGE_PARTITION_SPEC_ID_COLUMN}"
+                ))
+            })?
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "{MERGE_PARTITION_SPEC_ID_COLUMN} must be an Int32 column"
+                ))
+            })?;
+        let partitions = batch
+            .column_by_name(MERGE_PARTITION_COLUMN)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!("missing column {MERGE_PARTITION_COLUMN}"))
+            })?
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!("{MERGE_PARTITION_COLUMN} must be a Utf8 column"))
+            })?;
+
+        for row in 0..batch.num_rows() {
+            if file_paths.is_null(row) || row_indices.is_null(row) {
+                continue;
+            }
+            if partition_spec_ids.is_null(row) || partitions.is_null(row) {
+                return Err(DataFusionError::Plan(
+                    "MERGE position delete rows require Iceberg partition metadata".to_string(),
+                ));
+            }
+
+            let file_path = file_paths.value(row);
+            let partition_spec_id = partition_spec_ids.value(row);
+            let partition_json = partitions.value(row);
+            let rows = match self.rows_by_file.entry(file_path.to_string()) {
+                Entry::Occupied(entry) => {
+                    let rows = entry.into_mut();
+                    if rows.target.partition_spec_id != partition_spec_id
+                        || rows.target.partition_json != partition_json
+                    {
+                        return Err(DataFusionError::Plan(format!(
+                            "inconsistent Iceberg partition metadata for MERGE target file {file_path}"
+                        )));
+                    }
+                    rows
+                }
+                Entry::Vacant(entry) => entry.insert(PositionDeleteRows {
+                    target: position_delete_target(
+                        table_meta,
+                        file_path,
+                        partition_spec_id,
+                        partition_json,
+                    )?,
+                    positions: BTreeSet::new(),
+                }),
+            };
+
+            let row_index = row_indices.value(row);
+            if row_index < 0 {
+                return Err(DataFusionError::Plan(format!(
+                    "MERGE position delete row index must be non-negative, got {row_index}"
+                )));
+            }
+            rows.positions.insert(row_index);
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn finish(
+        self,
+        data_store_ctx: &StoreContext,
+        data_url: &Url,
+    ) -> Result> {
+        let mut delete_files = Vec::with_capacity(self.rows_by_file.len());
+        for rows in self.rows_by_file.into_values() {
+            delete_files.push(
+                write_position_delete_file(data_store_ctx, data_url, &rows.target, &rows.positions)
+                    .await?,
+            );
+        }
+        Ok(delete_files)
+    }
+}
+
+fn position_delete_target(
+    table_meta: &TableMetadata,
+    file_path: &str,
+    partition_spec_id: i32,
+    partition_json: &str,
+) -> Result {
+    if !table_meta
+        .partition_specs
+        .iter()
+        .any(|spec| spec.spec_id() == partition_spec_id)
+    {
+        return Err(DataFusionError::Plan(format!(
+            "MERGE target file uses unknown Iceberg partition spec {partition_spec_id}: {file_path}"
+        )));
+    }
+    let partition = serde_json::from_str(partition_json).map_err(|error| {
+        DataFusionError::Plan(format!(
+            "failed to decode Iceberg partition metadata for {file_path}: {error}"
+        ))
+    })?;
+    Ok(PositionDeleteTarget {
+        file_path: file_path.to_string(),
+        partition_spec_id,
+        partition_json: partition_json.to_string(),
+        partition,
+    })
+}
+
+const POSITION_DELETE_FILE_PATH_COL: &str = "file_path";
+const POSITION_DELETE_POS_COL: &str = "pos";
+const POSITION_DELETE_FILE_PATH_ID: &str = "2147483546";
+const POSITION_DELETE_POS_ID: &str = "2147483545";
+
+async fn write_position_delete_file(
+    data_store_ctx: &StoreContext,
+    data_url: &Url,
+    target: &PositionDeleteTarget,
+    positions: &BTreeSet,
+) -> Result {
+    let delete_schema = position_delete_arrow_schema();
+    let file_paths = (0..positions.len())
+        .map(|_| Some(target.file_path.as_str()))
+        .collect::>();
+    let pos_values = positions.iter().copied().collect::>();
+    let batch = ArrowRecordBatch::try_new(
+        Arc::new(delete_schema.clone()),
+        vec![
+            Arc::new(StringArray::from(file_paths)),
+            Arc::new(Int64Array::from(pos_values)),
+        ],
+    )?;
+
+    let mut writer = ArrowParquetWriter::try_new(&delete_schema, WriterProperties::default())
+        .map_err(DataFusionError::Execution)?;
+    writer
+        .write_batch(&batch)
+        .await
+        .map_err(DataFusionError::Execution)?;
+    let mut delete_file = delete_writer_common::write_delete_parquet_file(
+        data_store_ctx,
+        data_url,
+        "delete",
+        writer,
+        target.partition_spec_id,
+        target.partition.clone(),
+    )
+    .await?;
+    delete_file.content = DataContentType::PositionDeletes;
+    delete_file.referenced_data_file = Some(target.file_path.clone());
+    delete_file.sort_order_id = None;
+    delete_file.equality_ids.clear();
+    Ok(delete_file)
+}
+
+fn position_delete_arrow_schema() -> Schema {
+    Schema::new(vec![
+        Arc::new(
+            Field::new(POSITION_DELETE_FILE_PATH_COL, DataType::Utf8, false).with_metadata(
+                [(
+                    PARQUET_FIELD_ID_META_KEY.to_string(),
+                    POSITION_DELETE_FILE_PATH_ID.to_string(),
+                )]
+                .into_iter()
+                .collect(),
+            ),
+        ),
+        Arc::new(
+            Field::new(POSITION_DELETE_POS_COL, DataType::Int64, false).with_metadata(
+                [(
+                    PARQUET_FIELD_ID_META_KEY.to_string(),
+                    POSITION_DELETE_POS_ID.to_string(),
+                )]
+                .into_iter()
+                .collect(),
+            ),
+        ),
+    ])
+}
diff --git a/crates/sail-iceberg/src/physical_plan/scan_by_data_files_exec.rs b/crates/sail-iceberg/src/physical_plan/scan_by_data_files_exec.rs
index f117158c2..dcb750ed1 100644
--- a/crates/sail-iceberg/src/physical_plan/scan_by_data_files_exec.rs
+++ b/crates/sail-iceberg/src/physical_plan/scan_by_data_files_exec.rs
@@ -19,7 +19,7 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
     PlanProperties, SendableRecordBatchStream,
 };
-use datafusion_common::{internal_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, internal_err};
 use futures::stream::{self, StreamExt, TryStreamExt};
 use object_store::ObjectMeta;
 use sail_common_datafusion::schema_evolution::SchemaEvolutionPhysicalExprAdapterFactory;
diff --git a/crates/sail-iceberg/src/physical_plan/write_location.rs b/crates/sail-iceberg/src/physical_plan/write_location.rs
new file mode 100644
index 000000000..8a7e69eca
--- /dev/null
+++ b/crates/sail-iceberg/src/physical_plan/write_location.rs
@@ -0,0 +1,208 @@
+use std::collections::HashMap;
+
+use datafusion_common::{DataFusionError, Result};
+use url::Url;
+
+pub(crate) fn resolve_data_location_from_options_and_properties(
+    write_data_path: Option<&str>,
+    write_folder_storage_path: Option<&str>,
+    properties: &HashMap,
+    table_url: &Url,
+) -> Result {
+    if let Some(data_location) = resolve_data_location_from_property_value(
+        write_data_path.or(write_folder_storage_path),
+        table_url,
+    )? {
+        return Ok(data_location);
+    }
+    resolve_data_location_from_properties(properties, table_url)
+}
+
+pub(crate) fn resolve_data_location_from_property_value(
+    value: Option<&str>,
+    table_url: &Url,
+) -> Result> {
+    let Some(raw) = value.map(str::trim) else {
+        return Ok(None);
+    };
+    if raw.is_empty() {
+        return Ok(None);
+    }
+
+    let normalized_path = raw.replace('\\', "/");
+    let mut data_url = match (
+        crate::utils::parse_absolute_url(raw),
+        crate::utils::file_url_from_absolute_path(&normalized_path),
+    ) {
+        (Some(property_url), _) => property_url,
+        (None, Some(file_url)) => file_url,
+        (None, None) => table_url.join(&normalized_path).map_err(|error| {
+            DataFusionError::Plan(format!("Invalid Iceberg data path: {error}"))
+        })?,
+    };
+
+    let schemes_are_compatible = data_url.scheme() == table_url.scheme()
+        || matches!(
+            (table_url.scheme(), data_url.scheme()),
+            ("s3", "s3a") | ("s3a", "s3")
+        );
+    if !schemes_are_compatible || data_url.authority() != table_url.authority() {
+        return Err(DataFusionError::Plan(format!(
+            "Iceberg data path {data_url} must use the same object store as table location \
+             {table_url}"
+        )));
+    }
+
+    if !data_url.path().ends_with('/') {
+        data_url.set_path(&format!("{}/", data_url.path()));
+    }
+    Ok(Some(data_url))
+}
+
+pub(crate) fn resolve_data_location_from_properties(
+    properties: &HashMap,
+    table_url: &Url,
+) -> Result {
+    resolve_data_location_from_property_value(
+        properties
+            .get("write.data.path")
+            .or_else(|| properties.get("write.folder-storage.path"))
+            .map(String::as_str),
+        table_url,
+    )?
+    .map_or_else(
+        || {
+            table_url.join("data/").map_err(|error| {
+                DataFusionError::Plan(format!("Invalid default Iceberg data path: {error}"))
+            })
+        },
+        Ok,
+    )
+}
+
+pub(crate) fn parquet_file_name(file_prefix: &str) -> String {
+    format!("{file_prefix}-{}.parquet", uuid::Uuid::new_v4())
+}
+
+pub(crate) fn manifest_file_path(data_url: &Url, relative_path: &str) -> String {
+    data_url.join(&format!("./{relative_path}")).map_or_else(
+        |_| format!("{}{relative_path}", data_url.as_str()),
+        |url| url.to_string(),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn table_url() -> std::result::Result {
+        Url::parse("file:///tmp/iceberg/table/")
+    }
+
+    #[test]
+    fn data_location_prefers_write_data_path_option()
+    -> std::result::Result<(), Box> {
+        let properties =
+            HashMap::from([("write.data.path".to_string(), "property-data".to_string())]);
+        let table_url = table_url()?;
+
+        let actual = resolve_data_location_from_options_and_properties(
+            Some("option-data"),
+            Some("folder-data"),
+            &properties,
+            &table_url,
+        )?;
+
+        assert_eq!(
+            actual,
+            Url::parse("file:///tmp/iceberg/table/option-data/")?
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn data_location_uses_folder_storage_path_option_before_table_properties()
+    -> std::result::Result<(), Box> {
+        let properties =
+            HashMap::from([("write.data.path".to_string(), "property-data".to_string())]);
+        let table_url = table_url()?;
+
+        let actual = resolve_data_location_from_options_and_properties(
+            None,
+            Some("folder-data"),
+            &properties,
+            &table_url,
+        )?;
+
+        assert_eq!(
+            actual,
+            Url::parse("file:///tmp/iceberg/table/folder-data/")?
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn data_location_falls_back_to_table_properties_then_default()
+    -> std::result::Result<(), Box> {
+        let table_url = table_url()?;
+        let properties = HashMap::from([(
+            "write.folder-storage.path".to_string(),
+            "property-folder".to_string(),
+        )]);
+
+        let actual =
+            resolve_data_location_from_options_and_properties(None, None, &properties, &table_url)?;
+        assert_eq!(
+            actual,
+            Url::parse("file:///tmp/iceberg/table/property-folder/")?
+        );
+
+        let actual = resolve_data_location_from_options_and_properties(
+            None,
+            None,
+            &HashMap::new(),
+            &table_url,
+        )?;
+        assert_eq!(actual, Url::parse("file:///tmp/iceberg/table/data/")?);
+        Ok(())
+    }
+
+    #[test]
+    fn data_location_accepts_absolute_paths_outside_table_root()
+    -> std::result::Result<(), Box> {
+        let table_url = table_url()?;
+        let actual = resolve_data_location_from_property_value(
+            Some("file:///tmp/iceberg/external-data/"),
+            &table_url,
+        )?;
+
+        assert_eq!(
+            actual,
+            Some(Url::parse("file:///tmp/iceberg/external-data/")?)
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn data_location_accepts_s3_scheme_aliases() -> std::result::Result<(), url::ParseError> {
+        let table_url = Url::parse("s3://bucket/table/")?;
+        let result =
+            resolve_data_location_from_property_value(Some("s3a://bucket/data/"), &table_url);
+        let expected = Url::parse("s3a://bucket/data/")?;
+
+        assert!(matches!(result, Ok(Some(actual)) if actual == expected));
+        Ok(())
+    }
+
+    #[test]
+    fn data_location_rejects_different_object_store() -> std::result::Result<(), url::ParseError> {
+        let table_url = Url::parse("s3://table-bucket/table/")?;
+        let result =
+            resolve_data_location_from_property_value(Some("s3://data-bucket/data/"), &table_url);
+
+        assert!(
+            matches!(result, Err(DataFusionError::Plan(message)) if message.contains("must use the same object store"))
+        );
+        Ok(())
+    }
+}
diff --git a/crates/sail-iceberg/src/physical_plan/writer_exec.rs b/crates/sail-iceberg/src/physical_plan/writer_exec.rs
index 650d4fdfa..862b7d8d3 100644
--- a/crates/sail-iceberg/src/physical_plan/writer_exec.rs
+++ b/crates/sail-iceberg/src/physical_plan/writer_exec.rs
@@ -24,27 +24,34 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
     PlanProperties, SendableRecordBatchStream,
 };
-use datafusion_common::{internal_err, DataFusionError, Result};
-use futures::stream::once;
+use datafusion_common::{DataFusionError, Result, internal_err};
 use futures::StreamExt;
+use futures::stream::once;
 use parquet::file::properties::WriterProperties;
 use sail_common_datafusion::catalog::{CatalogPartitionField, LakehouseExecutionContext};
-use sail_common_datafusion::datasource::PhysicalSinkMode;
+use sail_common_datafusion::datasource::{
+    MERGE_FILE_COLUMN, MERGE_ROW_INDEX_COLUMN, PhysicalSinkMode,
+};
 use url::Url;
 
 use crate::datasource::type_converter::{arrow_schema_to_iceberg, iceberg_schema_to_arrow};
+use crate::io::StoreContext;
 use crate::operations::write::config::WriterConfig;
 use crate::operations::write::table_writer::IcebergTableWriter;
 use crate::physical_plan::action_schema::{
-    encode_add_data_files, encode_commit_meta, iceberg_action_schema, CommitMeta,
+    CommitMeta, encode_add_data_files, encode_commit_meta, encode_delete_data_files,
+    iceberg_action_schema,
 };
+use crate::physical_plan::merge_row_projection::IcebergMergeRowProjection;
+use crate::physical_plan::position_delete_writer::PositionDeleteAccumulator;
+use crate::physical_plan::write_location;
 use crate::physical_plan::writer_options::IcebergWriterExecOptions;
 use crate::schema_evolution::{SchemaEvolver, SchemaMode};
 use crate::spec::partition::{
     PartitionSpec as BoundPartitionSpec, UnboundPartitionField, UnboundPartitionSpec,
 };
 use crate::spec::schema::Schema as IcebergSchema;
-use crate::spec::{TableMetadata, TableRequirement};
+use crate::spec::{FormatVersion, TableMetadata, TableRequirement};
 use crate::table::metadata_loader::metadata_location_to_object_path_string;
 use crate::table_format::{
     catalog_managed_iceberg_from_properties, metadata_location_from_properties,
@@ -64,6 +71,7 @@ pub struct IcebergWriterExec {
     table_exists: bool,
     options: IcebergWriterExecOptions,
     logical_input_schema: Option,
+    merge_row_intents: bool,
     cache: Arc,
 }
 
@@ -108,7 +116,8 @@ impl IcebergWriterExec {
                 Arc::new(datafusion::arrow::datatypes::Schema::empty())
             }
         };
-        let cache = Self::compute_properties(schema.clone());
+        let output_partitions = input.output_partitioning().partition_count().max(1);
+        let cache = Self::compute_properties(schema.clone(), output_partitions);
         Self {
             input,
             table_url,
@@ -117,14 +126,40 @@ impl IcebergWriterExec {
             table_exists,
             options,
             logical_input_schema,
+            merge_row_intents: false,
             cache,
         }
     }
 
-    fn compute_properties(schema: datafusion::arrow::datatypes::SchemaRef) -> Arc {
+    pub fn new_merge(
+        input: Arc,
+        table_url: Url,
+        partition_columns: Vec,
+        sink_mode: PhysicalSinkMode,
+        table_exists: bool,
+        options: IcebergWriterExecOptions,
+        logical_input_schema: Option,
+    ) -> Self {
+        let mut writer = Self::new(
+            input,
+            table_url,
+            partition_columns,
+            sink_mode,
+            table_exists,
+            options,
+            logical_input_schema,
+        );
+        writer.merge_row_intents = true;
+        writer
+    }
+
+    fn compute_properties(
+        schema: datafusion::arrow::datatypes::SchemaRef,
+        output_partitions: usize,
+    ) -> Arc {
         Arc::new(PlanProperties::new(
             EquivalenceProperties::new(schema),
-            Partitioning::UnknownPartitioning(1),
+            Partitioning::UnknownPartitioning(output_partitions),
             EmissionType::Final,
             Boundedness::Bounded,
         ))
@@ -162,8 +197,11 @@ impl IcebergWriterExec {
         self.logical_input_schema.as_ref()
     }
 
-    fn input_schema_with_logical_metadata(&self) -> SchemaRef {
-        let physical_schema = self.input.schema();
+    pub fn reads_merge_row_intents(&self) -> bool {
+        self.merge_row_intents
+    }
+
+    fn input_schema_with_logical_metadata(&self, physical_schema: SchemaRef) -> SchemaRef {
         let Some(logical_schema) = self.logical_input_schema.as_ref() else {
             return physical_schema;
         };
@@ -215,76 +253,6 @@ impl IcebergWriterExec {
             (false, false) => Ok(None),
         }
     }
-
-    fn resolve_data_dir(table_meta: &TableMetadata, table_url: &Url) -> String {
-        Self::resolve_data_dir_from_properties(&table_meta.properties, table_url)
-    }
-
-    fn resolve_data_dir_from_property_value(
-        value: Option<&str>,
-        table_url: &Url,
-    ) -> Option {
-        let raw = value?.trim();
-        if raw.is_empty() {
-            return None;
-        }
-
-        let base_path = crate::utils::url_to_object_path(table_url).ok();
-        if let Some(prop_url) = crate::utils::parse_absolute_url(raw) {
-            if prop_url.scheme() == table_url.scheme()
-                && prop_url.host_str() == table_url.host_str()
-            {
-                if let (Ok(prop_path), Some(base_path)) = (
-                    crate::utils::url_to_object_path(&prop_url),
-                    base_path.as_ref(),
-                ) {
-                    let prop_str = prop_path.as_ref();
-                    let base_str = base_path.as_ref();
-                    if let Some(stripped) = prop_str.strip_prefix(base_str) {
-                        let rel = stripped.trim_start_matches('/').trim_matches('/');
-                        if !rel.is_empty() {
-                            return Some(rel.to_string());
-                        }
-                    }
-                }
-            }
-        } else {
-            let prop_path = raw.replace('\\', "/");
-            if prop_path.starts_with('/') {
-                if let Some(base_path) = base_path.as_ref() {
-                    let base_str = base_path.as_ref();
-                    let prop_no_leading = prop_path.trim_start_matches('/');
-                    if let Some(stripped) = prop_no_leading.strip_prefix(base_str) {
-                        let rel = stripped.trim_start_matches('/').trim_matches('/');
-                        if !rel.is_empty() {
-                            return Some(rel.to_string());
-                        }
-                    }
-                }
-            } else {
-                let rel = prop_path.trim_matches('/');
-                if !rel.is_empty() {
-                    return Some(rel.to_string());
-                }
-            }
-        }
-
-        None
-    }
-
-    fn resolve_data_dir_from_properties(
-        properties: &std::collections::HashMap,
-        table_url: &Url,
-    ) -> String {
-        Self::resolve_data_dir_from_property_value(
-            properties
-                .get("write.data.path")
-                .or_else(|| properties.get("write.folder-storage.path"))
-                .map(String::as_str),
-            table_url,
-        )
-        .unwrap_or_else(|| "data".to_string())
-    }
 }
 
 #[async_trait]
@@ -298,7 +266,7 @@ impl ExecutionPlan for IcebergWriterExec {
     }
 
     fn required_input_distribution(&self) -> Vec {
-        vec![Distribution::SinglePartition]
+        vec![Distribution::UnspecifiedDistribution]
     }
 
     fn children(&self) -> Vec<&Arc> {
@@ -312,15 +280,29 @@ impl ExecutionPlan for IcebergWriterExec {
         if children.len() != 1 {
             return internal_err!("IcebergWriterExec requires exactly one child");
         }
-        Ok(Arc::new(Self::new(
-            Arc::clone(&children[0]),
-            self.table_url.clone(),
-            self.partition_columns.clone(),
-            self.sink_mode.clone(),
-            self.table_exists,
-            self.options.clone(),
-            self.logical_input_schema.clone(),
-        )))
+        let input = Arc::clone(&children[0]);
+        let writer = if self.merge_row_intents {
+            Self::new_merge(
+                input,
+                self.table_url.clone(),
+                self.partition_columns.clone(),
+                self.sink_mode.clone(),
+                self.table_exists,
+                self.options.clone(),
+                self.logical_input_schema.clone(),
+            )
+        } else {
+            Self::new(
+                input,
+                self.table_url.clone(),
+                self.partition_columns.clone(),
+                self.sink_mode.clone(),
+                self.table_exists,
+                self.options.clone(),
+                self.logical_input_schema.clone(),
+            )
+        };
+        Ok(Arc::new(writer))
     }
 
     fn execute(
@@ -328,24 +310,37 @@ impl ExecutionPlan for IcebergWriterExec {
         partition: usize,
         context: Arc,
     ) -> Result {
-        if partition != 0 {
-            return internal_err!("IcebergWriterExec can only be executed in a single partition");
-        }
-
         let input_partitions = self.input.output_partitioning().partition_count();
-        if input_partitions != 1 {
+        if input_partitions == 0 {
+            return internal_err!("IcebergWriterExec requires at least one input partition");
+        }
+        if partition >= input_partitions {
             return internal_err!(
-                "IcebergWriterExec requires exactly one input partition, got {input_partitions}"
+                "IcebergWriterExec invalid partition {partition} (input partitions: {input_partitions})"
             );
         }
 
-        let stream = self.input.execute(0, Arc::clone(&context))?;
+        let stream = self.input.execute(partition, Arc::clone(&context))?;
 
         let table_url = self.table_url.clone();
         let partition_columns = self.partition_columns.clone();
         let sink_mode = self.sink_mode.clone();
         let table_exists = self.table_exists;
-        let input_schema = self.input_schema_with_logical_metadata();
+        let merge_projection = self
+            .merge_row_intents
+            .then(|| IcebergMergeRowProjection::try_new(self.input.schema()))
+            .transpose()?;
+        let writes_position_deletes = merge_projection.is_some()
+            && self
+                .input
+                .schema()
+                .field_with_name(MERGE_ROW_INDEX_COLUMN)
+                .is_ok();
+        let physical_input_schema = merge_projection
+            .as_ref()
+            .map(IcebergMergeRowProjection::data_schema)
+            .unwrap_or_else(|| self.input.schema());
+        let input_schema = self.input_schema_with_logical_metadata(physical_input_schema);
         let options = self.options.clone();
         let schema_mode = Self::get_schema_mode(&options, &sink_mode)?;
 
@@ -374,39 +369,50 @@ impl ExecutionPlan for IcebergWriterExec {
                 }
             }
 
-            let object_store = get_object_store_from_context(&context, &table_url)?;
+            let table_object_store = get_object_store_from_context(&context, &table_url)?;
             let input_schema = input_schema.clone();
 
             let (
                 iceberg_schema,
                 table_schema,
                 default_spec,
-                data_dir,
+                data_location,
                 spec_id_val,
                 commit_schema,
                 commit_requirements,
                 variant_shredding,
+                table_metadata,
             ) = if table_exists {
                 let latest_meta =
                     if catalog_managed_iceberg_from_properties(&options.table_properties) {
                         match metadata_location_from_properties(&options.table_properties) {
                             Some(location) => metadata_location_to_object_path_string(&location)?,
                             None => {
-                                crate::table::find_latest_metadata_file(&object_store, &table_url)
-                                    .await?
+                                crate::table::find_latest_metadata_file(
+                                    &table_object_store,
+                                    &table_url,
+                                )
+                                .await?
                             }
                         }
                     } else {
-                        crate::table::find_latest_metadata_file(&object_store, &table_url).await?
+                        crate::table::find_latest_metadata_file(&table_object_store, &table_url)
+                            .await?
                     };
                 let bytes = crate::table::metadata_loader::load_metadata_file_bytes(
-                    &object_store,
+                    &table_object_store,
                     &latest_meta,
                 )
                 .await?;
                 let table_meta = TableMetadata::from_json(&bytes)
                     .map_err(|e| DataFusionError::External(Box::new(e)))?;
-                let data_dir = Self::resolve_data_dir(&table_meta, &table_url);
+                let data_location =
+                    write_location::resolve_data_location_from_options_and_properties(
+                        options.write_data_path.as_deref(),
+                        options.write_folder_storage_path.as_deref(),
+                        &table_meta.properties,
+                        &table_url,
+                    )?;
                 let variant_shredding = options.variant_shredding_config(&table_meta.properties)?;
                 // FIXME: Concurrency Issue with Schema Evolution.
                 // This requires a mechanism to reserve Field IDs or restart the Writer task upon conflict.
@@ -476,11 +482,12 @@ impl ExecutionPlan for IcebergWriterExec {
                     schema_outcome.iceberg_schema,
                     schema_outcome.arrow_schema,
                     default_spec,
-                    data_dir,
+                    data_location,
                     spec_id_val,
                     commit_schema,
                     requirements,
                     variant_shredding,
+                    Some(table_meta),
                 )
             } else {
                 let (_, metadata_properties) =
@@ -520,20 +527,17 @@ impl ExecutionPlan for IcebergWriterExec {
                     iceberg_schema.clone(),
                     Arc::new(iceberg_schema_to_arrow(&iceberg_schema)?),
                     Some(spec),
-                    Self::resolve_data_dir_from_property_value(
-                        options
-                            .write_data_path
-                            .as_deref()
-                            .or(options.write_folder_storage_path.as_deref()),
+                    write_location::resolve_data_location_from_options_and_properties(
+                        options.write_data_path.as_deref(),
+                        options.write_folder_storage_path.as_deref(),
+                        &metadata_properties,
                         &table_url,
-                    )
-                    .unwrap_or_else(|| {
-                        Self::resolve_data_dir_from_properties(&metadata_properties, &table_url)
-                    }),
+                    )?,
                     sid,
                     Some(iceberg_schema),
                     Vec::new(),
                     variant_shredding,
+                    None,
                 )
             };
 
@@ -566,22 +570,71 @@ impl ExecutionPlan for IcebergWriterExec {
                 variant_shredding,
             };
 
-            let writer_root = crate::utils::url_to_object_path(&table_url)
+            let data_object_store = get_object_store_from_context(&context, &data_location)?;
+            let writer_root = crate::utils::url_to_object_path(&data_location)
                 .map_err(|e| DataFusionError::Plan(e.to_string()))?;
             let mut writer = IcebergTableWriter::new(
-                object_store.clone(),
+                data_object_store.clone(),
                 writer_root,
                 writer_config,
                 spec_id_val,
-                data_dir,
-                table_url.clone(),
+                data_location.clone(),
             );
 
+            let mut position_deletes = if writes_position_deletes {
+                let table_metadata = table_metadata.as_ref().ok_or_else(|| {
+                    DataFusionError::Internal(
+                        "Iceberg MERGE position deletes require existing table metadata"
+                            .to_string(),
+                    )
+                })?;
+                match table_metadata.format_version {
+                    FormatVersion::V1 => {
+                        return Err(DataFusionError::Plan(
+                            "Iceberg position delete writes require table format-version 2"
+                                .to_string(),
+                        ));
+                    }
+                    FormatVersion::V2 => {}
+                    FormatVersion::V3 => {
+                        return Err(DataFusionError::NotImplemented(
+                            "Iceberg v3 MERGE MOR position delete writes are not supported; v3 requires deletion vectors".to_string(),
+                        ));
+                    }
+                }
+                Some(PositionDeleteAccumulator::default())
+            } else {
+                None
+            };
+
             let mut total_rows = 0u64;
             let mut data = stream;
             while let Some(batch_result) = data.next().await {
-                let batch = batch_result?;
+                let input_batch = batch_result?;
+                let batch = if let Some(merge_projection) = &merge_projection {
+                    if let Some(position_deletes) = &mut position_deletes {
+                        let table_metadata = table_metadata.as_ref().ok_or_else(|| {
+                            DataFusionError::Internal(
+                                "Iceberg MERGE position deletes require table metadata".to_string(),
+                            )
+                        })?;
+                        let delete_rows =
+                            merge_projection.project_position_delete_rows(&input_batch)?;
+                        position_deletes.add_batch(
+                            table_metadata,
+                            &delete_rows,
+                            MERGE_FILE_COLUMN,
+                            MERGE_ROW_INDEX_COLUMN,
+                        )?;
+                    }
+                    merge_projection.project_data_rows(&input_batch)?
+                } else {
+                    input_batch
+                };
                 let batch_row_count = batch.num_rows();
+                if batch_row_count == 0 {
+                    continue;
+                }
                 total_rows += u64::try_from(batch_row_count).map_err(|e| {
                     DataFusionError::Execution(format!("Row count overflow: {}", e))
                 })?;
@@ -592,15 +645,18 @@ impl ExecutionPlan for IcebergWriterExec {
             }
 
             let data_files = writer.close().await.map_err(DataFusionError::Execution)?;
+            let delete_files = if let Some(position_deletes) = position_deletes {
+                let data_store_ctx = StoreContext::new(data_object_store, &data_location)?;
+                position_deletes
+                    .finish(&data_store_ctx, &data_location)
+                    .await?
+            } else {
+                Vec::new()
+            };
 
             let commit_meta = CommitMeta {
                 table_uri: table_url.to_string(),
                 row_count: total_rows,
-                operation: if matches!(sink_mode, PhysicalSinkMode::Overwrite) {
-                    crate::spec::Operation::Overwrite
-                } else {
-                    crate::spec::Operation::Append
-                },
                 requirements: commit_requirements,
                 table_properties: options.table_properties,
                 lakehouse_table: options.lakehouse_table,
@@ -617,6 +673,7 @@ impl ExecutionPlan for IcebergWriterExec {
             let schema = iceberg_action_schema()?;
             let batches = vec![
                 encode_add_data_files(data_files)?,
+                encode_delete_data_files(delete_files)?,
                 encode_commit_meta(commit_meta)?,
             ];
             let batch = concat_batches(&schema, &batches)
@@ -636,11 +693,19 @@ impl DisplayAs for IcebergWriterExec {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
-                write!(f, "IcebergWriterExec(table_path={})", self.table_url)
+                write!(f, "IcebergWriterExec(table_path={}", self.table_url)?;
+                if self.merge_row_intents {
+                    write!(f, ", merge_row_intents=true")?;
+                }
+                write!(f, ")")
             }
             DisplayFormatType::TreeRender => {
                 writeln!(f, "format: iceberg")?;
-                write!(f, "table_path={}", self.table_url)
+                write!(f, "table_path={}", self.table_url)?;
+                if self.merge_row_intents {
+                    write!(f, ", merge_row_intents=true")?;
+                }
+                Ok(())
             }
         }
     }
diff --git a/crates/sail-iceberg/src/physical_plan/writer_options.rs b/crates/sail-iceberg/src/physical_plan/writer_options.rs
index f325f082d..199698213 100644
--- a/crates/sail-iceberg/src/physical_plan/writer_options.rs
+++ b/crates/sail-iceberg/src/physical_plan/writer_options.rs
@@ -19,7 +19,7 @@ use sail_common_datafusion::variant::DEFAULT_VARIANT_INFERENCE_NODE_BUDGET;
 use serde::{Deserialize, Serialize};
 
 use crate::operations::write::config::VariantShreddingConfig;
-use crate::options::gen::IcebergWriteOptions;
+use crate::options::r#gen::IcebergWriteOptions;
 
 const PARQUET_SHRED_VARIANTS: &str = "write.parquet.shred-variants";
 const PARQUET_VARIANT_INFERENCE_BUFFER_SIZE: &str = "write.parquet.variant-inference-buffer-size";
diff --git a/crates/sail-iceberg/src/properties.rs b/crates/sail-iceberg/src/properties.rs
index 02d40464b..556fcb86c 100644
--- a/crates/sail-iceberg/src/properties.rs
+++ b/crates/sail-iceberg/src/properties.rs
@@ -1,8 +1,8 @@
 use std::collections::HashMap;
 
-use datafusion::common::{plan_err, DataFusionError, Result};
+use datafusion::common::{DataFusionError, Result, plan_err};
 use sail_common_datafusion::catalog::managed::{
-    is_metadata_location_key, PREVIOUS_METADATA_LOCATION_KEY,
+    PREVIOUS_METADATA_LOCATION_KEY, is_metadata_location_key,
 };
 
 use crate::spec::{FormatVersion, TableMetadata};
diff --git a/crates/sail-iceberg/src/row_level_metadata.rs b/crates/sail-iceberg/src/row_level_metadata.rs
new file mode 100644
index 000000000..f88e449b1
--- /dev/null
+++ b/crates/sail-iceberg/src/row_level_metadata.rs
@@ -0,0 +1,129 @@
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+use datafusion_common::{Result, plan_err};
+
+pub(crate) const MERGE_PARTITION_SPEC_ID_COLUMN: &str = "__sail_iceberg_partition_spec_id";
+pub(crate) const MERGE_PARTITION_COLUMN: &str = "__sail_iceberg_partition";
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct RowLevelMetadataColumns<'a> {
+    file_column_name: Option<&'a str>,
+    row_index_column_name: Option<&'a str>,
+    include_delete_file_metadata: bool,
+}
+
+impl<'a> RowLevelMetadataColumns<'a> {
+    pub(crate) fn new(
+        file_column_name: Option<&'a str>,
+        row_index_column_name: Option<&'a str>,
+    ) -> Self {
+        Self {
+            file_column_name,
+            row_index_column_name,
+            include_delete_file_metadata: false,
+        }
+    }
+
+    pub(crate) fn with_delete_file_metadata(mut self) -> Self {
+        self.include_delete_file_metadata = true;
+        self
+    }
+
+    pub(crate) fn append_to_schema(&self, data_schema: &ArrowSchema) -> Result {
+        self.validate_no_collisions(data_schema)?;
+        let mut fields = data_schema.fields().iter().cloned().collect::>();
+        if let Some(name) = self.file_column_name {
+            fields.push(Arc::new(Field::new(name, DataType::Utf8, true)));
+        }
+        if self.include_delete_file_metadata {
+            fields.push(Arc::new(Field::new(
+                MERGE_PARTITION_SPEC_ID_COLUMN,
+                DataType::Int32,
+                false,
+            )));
+            fields.push(Arc::new(Field::new(
+                MERGE_PARTITION_COLUMN,
+                DataType::Utf8,
+                false,
+            )));
+        }
+        if let Some(name) = self.row_index_column_name {
+            fields.push(Arc::new(Field::new(name, DataType::Int64, true)));
+        }
+        Ok(ArrowSchema::new_with_metadata(
+            fields,
+            data_schema.metadata().clone(),
+        ))
+    }
+
+    fn validate_no_collisions(&self, data_schema: &ArrowSchema) -> Result<()> {
+        let delete_file_metadata_columns = self
+            .include_delete_file_metadata
+            .then_some([MERGE_PARTITION_SPEC_ID_COLUMN, MERGE_PARTITION_COLUMN]);
+        for name in [self.file_column_name, self.row_index_column_name]
+            .into_iter()
+            .flatten()
+            .chain(delete_file_metadata_columns.into_iter().flatten())
+        {
+            if data_schema.field_with_name(name).is_ok() {
+                return plan_err!(
+                    "Iceberg row-level metadata column '{name}' conflicts with an existing table column"
+                );
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::*;
+
+    #[test]
+    fn appends_metadata_columns_and_preserves_schema_metadata() -> Result<()> {
+        let schema = ArrowSchema::new_with_metadata(
+            vec![Arc::new(Field::new("id", DataType::Int64, false))],
+            HashMap::from([("owner".to_string(), "iceberg".to_string())]),
+        );
+
+        let actual =
+            RowLevelMetadataColumns::new(Some("__sail_file_path"), Some("__sail_file_row_index"))
+                .append_to_schema(&schema)?;
+
+        assert_eq!(
+            actual.metadata().get("owner").map(String::as_str),
+            Some("iceberg")
+        );
+        assert_eq!(actual.fields().len(), 3);
+        assert_eq!(actual.field(1).name(), "__sail_file_path");
+        assert_eq!(actual.field(1).data_type(), &DataType::Utf8);
+        assert_eq!(actual.field(2).name(), "__sail_file_row_index");
+        assert_eq!(actual.field(2).data_type(), &DataType::Int64);
+        Ok(())
+    }
+
+    #[test]
+    fn rejects_existing_metadata_column_names() -> Result<()> {
+        let schema = ArrowSchema::new(vec![Arc::new(Field::new(
+            "__sail_file_path",
+            DataType::Utf8,
+            true,
+        ))]);
+
+        let err = match RowLevelMetadataColumns::new(Some("__sail_file_path"), None)
+            .append_to_schema(&schema)
+        {
+            Ok(_) => return plan_err!("metadata column conflict should fail"),
+            Err(e) => e,
+        };
+
+        assert!(
+            err.to_string()
+                .contains("conflicts with an existing table column")
+        );
+        Ok(())
+    }
+}
diff --git a/crates/sail-iceberg/src/schema_evolution.rs b/crates/sail-iceberg/src/schema_evolution.rs
index 97bc46ae2..dc4cd6424 100644
--- a/crates/sail-iceberg/src/schema_evolution.rs
+++ b/crates/sail-iceberg/src/schema_evolution.rs
@@ -18,9 +18,9 @@ use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Schema as ArrowSch
 use datafusion_common::{DataFusionError, Result};
 use sail_common_datafusion::variant::variant_storage_types_equivalent;
 
+use crate::spec::TableMetadata;
 use crate::spec::schema::{Schema as IcebergSchema, SchemaBuilder};
 use crate::spec::types::{ListType, MapType, NestedField, PrimitiveType, StructType, Type};
-use crate::spec::TableMetadata;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum SchemaMode {
@@ -223,10 +223,10 @@ impl SchemaEvolver {
             DataType::Timestamp(table_unit, table_tz),
             DataType::Timestamp(input_unit, input_tz),
         ) = (table_type, input_type)
+            && table_unit == input_unit
+            && Self::timestamp_timezone_compatible(table_tz, input_tz)
         {
-            if table_unit == input_unit && Self::timestamp_timezone_compatible(table_tz, input_tz) {
-                return true;
-            }
+            return true;
         }
         match (table_type, input_type) {
             (DataType::Struct(table_fields), DataType::Struct(input_fields)) => {
@@ -478,19 +478,22 @@ impl SchemaEvolver {
 
         // 1. Build fields in the order of Input Arrow Schema
         for input_field in input_fields {
-            if let Some(existing_field) = existing_pool.remove(input_field.name()) {
-                // Field exists: perform recursive merge_field, preserving original ID
-                let (merged_field, field_changed) =
-                    Self::merge_field(existing_field.as_ref(), input_field, next_field_id)?;
-                changed |= field_changed;
-                merged_children.push(Arc::new(merged_field));
-            } else {
-                // FIXME: - Missing Nested Column Rename detection.
-                // Same as top-level fields: renaming a nested field is currently treated as adding a new field.
-                // This breaks schema evolution for renamed nested fields.
-                let new_field = Self::build_field_from_arrow(input_field, next_field_id)?;
-                merged_children.push(Arc::new(new_field));
-                changed = true;
+            match existing_pool.remove(input_field.name()) {
+                Some(existing_field) => {
+                    // Field exists: perform recursive merge_field, preserving original ID
+                    let (merged_field, field_changed) =
+                        Self::merge_field(existing_field.as_ref(), input_field, next_field_id)?;
+                    changed |= field_changed;
+                    merged_children.push(Arc::new(merged_field));
+                }
+                _ => {
+                    // FIXME: - Missing Nested Column Rename detection.
+                    // Same as top-level fields: renaming a nested field is currently treated as adding a new field.
+                    // This breaks schema evolution for renamed nested fields.
+                    let new_field = Self::build_field_from_arrow(input_field, next_field_id)?;
+                    merged_children.push(Arc::new(new_field));
+                    changed = true;
+                }
             }
         }
 
@@ -843,8 +846,8 @@ mod tests {
     use crate::spec::metadata::format::FormatVersion;
     use crate::spec::partition::PartitionSpec;
     use crate::spec::transform::Transform;
-    use crate::spec::types::values::{Literal, PrimitiveLiteral};
     use crate::spec::types::PrimitiveType;
+    use crate::spec::types::values::{Literal, PrimitiveLiteral};
 
     #[test]
     fn assign_schema_field_ids_assigns_nested_children() {
diff --git a/crates/sail-iceberg/src/spec/avro_utils.rs b/crates/sail-iceberg/src/spec/avro_utils.rs
index da7e126fd..aa97edcaa 100644
--- a/crates/sail-iceberg/src/spec/avro_utils.rs
+++ b/crates/sail-iceberg/src/spec/avro_utils.rs
@@ -10,8 +10,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use apache_avro::schema::{RecordField as AvroRecordField, RecordFieldOrder, UnionSchema};
 use apache_avro::Schema as AvroSchema;
+use apache_avro::schema::{RecordField as AvroRecordField, RecordFieldOrder, UnionSchema};
 use serde_json::{Number, Value as JsonValue};
 
 /// Avro custom attribute used to annotate Iceberg field ids.
diff --git a/crates/sail-iceberg/src/spec/catalog/mod.rs b/crates/sail-iceberg/src/spec/catalog/mod.rs
index e8b948010..8e699682c 100644
--- a/crates/sail-iceberg/src/spec/catalog/mod.rs
+++ b/crates/sail-iceberg/src/spec/catalog/mod.rs
@@ -376,10 +376,13 @@ mod _serde_set_statistics {
             snapshot_id,
             statistics,
         } = SetStatistics::deserialize(deserializer)?;
-        if let Some(snapshot_id) = snapshot_id {
-            if snapshot_id != statistics.snapshot_id {
-                return Err(serde::de::Error::custom(format!("Snapshot id to set {snapshot_id} does not match the statistics file snapshot id {}", statistics.snapshot_id)));
-            }
+        if let Some(snapshot_id) = snapshot_id
+            && snapshot_id != statistics.snapshot_id
+        {
+            return Err(serde::de::Error::custom(format!(
+                "Snapshot id to set {snapshot_id} does not match the statistics file snapshot id {}",
+                statistics.snapshot_id
+            )));
         }
         Ok(statistics)
     }
diff --git a/crates/sail-iceberg/src/spec/manifest/_serde.rs b/crates/sail-iceberg/src/spec/manifest/_serde.rs
index 0fcac3f7f..557d18fe2 100644
--- a/crates/sail-iceberg/src/spec/manifest/_serde.rs
+++ b/crates/sail-iceberg/src/spec/manifest/_serde.rs
@@ -23,8 +23,8 @@ use serde::de::{DeserializeOwned, IgnoredAny};
 use serde::{Deserialize, Deserializer, Serialize};
 
 use super::{DataContentType, DataFileFormat};
-use crate::spec::types::{Datum, PrimitiveLiteral, RawLiteral, StructType, Type};
 use crate::spec::Schema;
+use crate::spec::types::{Datum, PrimitiveLiteral, RawLiteral, StructType, Type};
 
 #[derive(Serialize, Deserialize)]
 pub(super) struct IntLongMapEntry {
diff --git a/crates/sail-iceberg/src/spec/manifest/data_file.rs b/crates/sail-iceberg/src/spec/manifest/data_file.rs
index 757aa09ec..846338483 100644
--- a/crates/sail-iceberg/src/spec/manifest/data_file.rs
+++ b/crates/sail-iceberg/src/spec/manifest/data_file.rs
@@ -21,8 +21,8 @@ use std::collections::HashMap;
 
 use serde::{Deserialize, Serialize};
 
-use crate::spec::types::values::Literal;
 use crate::spec::Datum;
+use crate::spec::types::values::Literal;
 
 /// Content type of a data file.
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize)]
diff --git a/crates/sail-iceberg/src/spec/manifest/mod.rs b/crates/sail-iceberg/src/spec/manifest/mod.rs
index d31318465..f1ec55f1e 100644
--- a/crates/sail-iceberg/src/spec/manifest/mod.rs
+++ b/crates/sail-iceberg/src/spec/manifest/mod.rs
@@ -19,7 +19,7 @@
 
 use std::sync::Arc;
 
-use apache_avro::{from_value as avro_from_value, Reader as AvroReader};
+use apache_avro::{Reader as AvroReader, from_value as avro_from_value};
 
 mod _serde;
 mod data_file;
@@ -29,15 +29,15 @@ mod schema;
 mod writer;
 
 // Provide data file avro helpers API surface
-use apache_avro::{to_value, Writer as AvroWriter};
+use apache_avro::{Writer as AvroWriter, to_value};
 pub use data_file::*;
 pub use entry::*;
 pub use metadata::*;
 pub use writer::*;
 
+use crate::spec::Schema as IcebergSchema;
 use crate::spec::metadata::format::FormatVersion;
 use crate::spec::types::StructType;
-use crate::spec::Schema as IcebergSchema;
 
 /// Convert data files to avro bytes and write to writer. Return the bytes written.
 pub fn write_data_files_to_avro(
diff --git a/crates/sail-iceberg/src/spec/manifest/schema.rs b/crates/sail-iceberg/src/spec/manifest/schema.rs
index 59a31b5a0..601cb7517 100644
--- a/crates/sail-iceberg/src/spec/manifest/schema.rs
+++ b/crates/sail-iceberg/src/spec/manifest/schema.rs
@@ -12,14 +12,14 @@
 
 use std::collections::BTreeMap;
 
+use apache_avro::Schema as AvroSchema;
 use apache_avro::schema::{
     ArraySchema, DecimalSchema, FixedSchema, Name, RecordField as AvroRecordField,
     RecordFieldOrder, RecordSchema,
 };
-use apache_avro::Schema as AvroSchema;
 use serde_json::{Number, Value as JsonValue};
 
-use crate::spec::avro_utils::{optional, record_field, FIELD_ID_ATTR};
+use crate::spec::avro_utils::{FIELD_ID_ATTR, optional, record_field};
 use crate::spec::types::{PrimitiveType, StructType, Type};
 
 const ELEMENT_ID: &str = "element-id";
@@ -247,11 +247,7 @@ fn array_of_longs(element_id: i32, required: bool) -> AvroSchema {
         items: Box::new(AvroSchema::Long),
         attributes: attrs,
     });
-    if required {
-        array
-    } else {
-        optional(array)
-    }
+    if required { array } else { optional(array) }
 }
 
 fn array_of_ints(element_id: i32, required: bool) -> AvroSchema {
@@ -264,11 +260,7 @@ fn array_of_ints(element_id: i32, required: bool) -> AvroSchema {
         items: Box::new(AvroSchema::Int),
         attributes: attrs,
     });
-    if required {
-        array
-    } else {
-        optional(array)
-    }
+    if required { array } else { optional(array) }
 }
 
 fn int_key_map(
diff --git a/crates/sail-iceberg/src/spec/manifest/writer.rs b/crates/sail-iceberg/src/spec/manifest/writer.rs
index 342d397af..22a027b83 100644
--- a/crates/sail-iceberg/src/spec/manifest/writer.rs
+++ b/crates/sail-iceberg/src/spec/manifest/writer.rs
@@ -19,13 +19,13 @@
 
 use std::sync::Arc;
 
-use apache_avro::{to_value, Writer as AvroWriter};
+use apache_avro::{Writer as AvroWriter, to_value};
 
 use super::{
     DataFile, Manifest, ManifestEntry, ManifestEntryRef, ManifestMetadata, ManifestStatus,
 };
-use crate::spec::manifest_list::{ManifestContentType, ManifestFile};
 use crate::spec::FormatVersion;
+use crate::spec::manifest_list::{ManifestContentType, ManifestFile};
 
 #[derive(Debug, Clone)]
 pub struct ManifestWriterBuilder {
diff --git a/crates/sail-iceberg/src/spec/manifest_list.rs b/crates/sail-iceberg/src/spec/manifest_list.rs
index 0f4c5dba5..b702a2452 100644
--- a/crates/sail-iceberg/src/spec/manifest_list.rs
+++ b/crates/sail-iceberg/src/spec/manifest_list.rs
@@ -18,7 +18,7 @@
 // [CREDIT]: https://raw.githubusercontent.com/apache/iceberg-rust/dc349284a4204c1a56af47fb3177ace6f9e899a0/crates/iceberg/src/spec/manifest_list.rs
 
 use apache_avro::types::Value as AvroValue;
-use apache_avro::{from_value as avro_from_value, Reader as AvroReader};
+use apache_avro::{Reader as AvroReader, from_value as avro_from_value};
 use serde::{Deserialize, Serialize};
 
 use crate::spec::FormatVersion;
@@ -26,6 +26,24 @@ mod schema;
 
 pub const UNASSIGNED_SEQUENCE_NUMBER: i64 = -1;
 
+fn optional_avro_int(value: &AvroValue) -> Result, String> {
+    match value {
+        AvroValue::Int(value) => Ok(Some(*value)),
+        AvroValue::Union(_, value) => optional_avro_int(value.as_ref()),
+        AvroValue::Null => Ok(None),
+        _ => Err("int or null".to_string()),
+    }
+}
+
+fn optional_avro_long(value: &AvroValue) -> Result, String> {
+    match value {
+        AvroValue::Long(value) => Ok(Some(*value)),
+        AvroValue::Union(_, value) => optional_avro_long(value.as_ref()),
+        AvroValue::Null => Ok(None),
+        _ => Err("long or null".to_string()),
+    }
+}
+
 /// Snapshots are embedded in table metadata, but the list of manifests for a
 /// snapshot are stored in a separate manifest list file.
 ///
@@ -86,7 +104,9 @@ impl ManifestList {
                             if let Ok(mf) = Self::parse_manifest_v2_fallback(&value) {
                                 entries.push(mf);
                             } else {
-                                let err = format!("Avro decode error: Failed to deserialize Avro value into value: {value:?}");
+                                let err = format!(
+                                    "Avro decode error: Failed to deserialize Avro value into value: {value:?}"
+                                );
                                 return Err(err);
                             }
                         }
@@ -107,7 +127,9 @@ impl ManifestList {
                                 if let Ok(mf) = Self::parse_manifest_v2_fallback(&value) {
                                     entries.push(mf);
                                 } else {
-                                    let err = format!("Avro decode error: Failed to deserialize Avro value into value: {value:?}");
+                                    let err = format!(
+                                        "Avro decode error: Failed to deserialize Avro value into value: {value:?}"
+                                    );
                                     return Err(err);
                                 }
                             }
@@ -299,46 +321,73 @@ impl ManifestList {
                 let manifest_path = string(get("manifest_path").ok_or("manifest_path")?)?;
                 let manifest_length = long(get("manifest_length").ok_or("manifest_length")?)?;
                 let partition_spec_id = int(get("partition_spec_id").ok_or("partition_spec_id")?)?;
-                let content = int(get("content").unwrap_or(&AvroValue::Int(0)))?;
-                let sequence_number = long(get("sequence_number").ok_or("sequence_number")?)?;
-                let min_sequence_number =
-                    long(get("min_sequence_number").ok_or("min_sequence_number")?)?;
+                let v2_field_presence = [
+                    get("content").is_some(),
+                    get("sequence_number").is_some(),
+                    get("min_sequence_number").is_some(),
+                ];
+                let is_v1_shape = v2_field_presence.iter().all(|present| !present);
+                if !is_v1_shape && !v2_field_presence.iter().all(|present| *present) {
+                    return Err("incomplete V2 manifest list tracking fields".to_string());
+                }
+                let content = match get("content") {
+                    Some(value) => optional_avro_int(value)?.ok_or("content cannot be null")?,
+                    None => 0,
+                };
+                let sequence_number = match get("sequence_number") {
+                    Some(value) => {
+                        optional_avro_long(value)?.ok_or("sequence_number cannot be null")?
+                    }
+                    None => 0,
+                };
+                let min_sequence_number = match get("min_sequence_number") {
+                    Some(value) => {
+                        optional_avro_long(value)?.ok_or("min_sequence_number cannot be null")?
+                    }
+                    None => 0,
+                };
                 let added_snapshot_id = long(get("added_snapshot_id").ok_or("added_snapshot_id")?)?;
                 let added_files_count = get("added_files_count")
                     .or_else(|| get("added_data_files_count"))
-                    .and_then(|v| {
-                        if let AvroValue::Int(x) = v {
-                            Some(*x)
-                        } else {
-                            None
-                        }
-                    })
-                    .unwrap_or(0);
+                    .map(optional_avro_int)
+                    .transpose()?
+                    .flatten();
                 let existing_files_count = get("existing_files_count")
                     .or_else(|| get("existing_data_files_count"))
-                    .and_then(|v| {
-                        if let AvroValue::Int(x) = v {
-                            Some(*x)
-                        } else {
-                            None
-                        }
-                    })
-                    .unwrap_or(0);
+                    .map(optional_avro_int)
+                    .transpose()?
+                    .flatten();
                 let deleted_files_count = get("deleted_files_count")
                     .or_else(|| get("deleted_data_files_count"))
-                    .and_then(|v| {
-                        if let AvroValue::Int(x) = v {
-                            Some(*x)
-                        } else {
-                            None
-                        }
-                    })
-                    .unwrap_or(0);
-                let added_rows_count = long(get("added_rows_count").ok_or("added_rows_count")?)?;
-                let existing_rows_count =
-                    long(get("existing_rows_count").ok_or("existing_rows_count")?)?;
-                let deleted_rows_count =
-                    long(get("deleted_rows_count").ok_or("deleted_rows_count")?)?;
+                    .map(optional_avro_int)
+                    .transpose()?
+                    .flatten();
+                let added_rows_count = get("added_rows_count")
+                    .map(optional_avro_long)
+                    .transpose()?
+                    .flatten();
+                let existing_rows_count = get("existing_rows_count")
+                    .map(optional_avro_long)
+                    .transpose()?
+                    .flatten();
+                let deleted_rows_count = get("deleted_rows_count")
+                    .map(optional_avro_long)
+                    .transpose()?
+                    .flatten();
+                if !is_v1_shape
+                    && [
+                        added_files_count.is_some(),
+                        existing_files_count.is_some(),
+                        deleted_files_count.is_some(),
+                        added_rows_count.is_some(),
+                        existing_rows_count.is_some(),
+                        deleted_rows_count.is_some(),
+                    ]
+                    .iter()
+                    .any(|present| !present)
+                {
+                    return Err("missing required V2 manifest list counts or rows".to_string());
+                }
 
                 let partitions = match get("partitions") {
                     Some(AvroValue::Union(_, inner)) => match inner.as_ref() {
@@ -413,12 +462,12 @@ impl ManifestList {
                     sequence_number,
                     min_sequence_number,
                     added_snapshot_id,
-                    added_files_count: Some(added_files_count),
-                    existing_files_count: Some(existing_files_count),
-                    deleted_files_count: Some(deleted_files_count),
-                    added_rows_count: Some(added_rows_count),
-                    existing_rows_count: Some(existing_rows_count),
-                    deleted_rows_count: Some(deleted_rows_count),
+                    added_files_count,
+                    existing_files_count,
+                    deleted_files_count,
+                    added_rows_count,
+                    existing_rows_count,
+                    deleted_rows_count,
                     partitions,
                     key_metadata,
                     first_row_id,
@@ -925,3 +974,91 @@ impl From<_serde::ManifestFileV1> for ManifestFile {
         }
     }
 }
+
+#[cfg(test)]
+#[expect(clippy::expect_used)]
+mod tests {
+    use super::*;
+
+    fn v1_manifest_list_record_with_unknown_counts() -> AvroValue {
+        AvroValue::Record(vec![
+            (
+                "manifest_path".to_string(),
+                AvroValue::String("metadata/manifest.avro".to_string()),
+            ),
+            ("manifest_length".to_string(), AvroValue::Long(10)),
+            ("partition_spec_id".to_string(), AvroValue::Int(0)),
+            ("added_snapshot_id".to_string(), AvroValue::Long(7)),
+            (
+                "added_data_files_count".to_string(),
+                AvroValue::Union(0, Box::new(AvroValue::Null)),
+            ),
+            (
+                "existing_data_files_count".to_string(),
+                AvroValue::Union(0, Box::new(AvroValue::Null)),
+            ),
+            (
+                "deleted_data_files_count".to_string(),
+                AvroValue::Union(0, Box::new(AvroValue::Null)),
+            ),
+            (
+                "added_rows_count".to_string(),
+                AvroValue::Union(0, Box::new(AvroValue::Null)),
+            ),
+            (
+                "existing_rows_count".to_string(),
+                AvroValue::Union(0, Box::new(AvroValue::Null)),
+            ),
+            (
+                "deleted_rows_count".to_string(),
+                AvroValue::Union(0, Box::new(AvroValue::Null)),
+            ),
+        ])
+    }
+
+    #[test]
+    fn v2_fallback_preserves_unknown_v1_counts() {
+        let manifest = ManifestList::parse_manifest_v2_fallback(
+            &v1_manifest_list_record_with_unknown_counts(),
+        )
+        .expect("V1 manifest list entry");
+
+        assert_eq!(manifest.content, ManifestContentType::Data);
+        assert_eq!(manifest.sequence_number, 0);
+        assert_eq!(manifest.min_sequence_number, 0);
+        assert_eq!(manifest.added_files_count, None);
+        assert_eq!(manifest.existing_files_count, None);
+        assert_eq!(manifest.deleted_files_count, None);
+        assert_eq!(manifest.added_rows_count, None);
+        assert_eq!(manifest.existing_rows_count, None);
+        assert_eq!(manifest.deleted_rows_count, None);
+    }
+
+    #[test]
+    fn v2_fallback_rejects_malformed_present_fields() {
+        let AvroValue::Record(mut fields) = v1_manifest_list_record_with_unknown_counts() else {
+            unreachable!("fixture is a record")
+        };
+        fields.push((
+            "sequence_number".to_string(),
+            AvroValue::String("not-a-sequence".to_string()),
+        ));
+
+        assert!(ManifestList::parse_manifest_v2_fallback(&AvroValue::Record(fields)).is_err());
+
+        let AvroValue::Record(mut nullable_v2_fields) =
+            v1_manifest_list_record_with_unknown_counts()
+        else {
+            unreachable!("fixture is a record")
+        };
+        nullable_v2_fields.extend([
+            ("content".to_string(), AvroValue::Int(0)),
+            ("sequence_number".to_string(), AvroValue::Long(3)),
+            ("min_sequence_number".to_string(), AvroValue::Long(3)),
+        ]);
+        assert!(
+            ManifestList::parse_manifest_v2_fallback(&AvroValue::Record(nullable_v2_fields))
+                .is_err()
+        );
+    }
+}
diff --git a/crates/sail-iceberg/src/spec/manifest_list/schema.rs b/crates/sail-iceberg/src/spec/manifest_list/schema.rs
index 45c71ca70..4a1e27272 100644
--- a/crates/sail-iceberg/src/spec/manifest_list/schema.rs
+++ b/crates/sail-iceberg/src/spec/manifest_list/schema.rs
@@ -12,8 +12,8 @@
 
 use std::collections::BTreeMap;
 
-use apache_avro::schema::{ArraySchema, Name, RecordField as AvroRecordField, RecordSchema};
 use apache_avro::Schema as AvroSchema;
+use apache_avro::schema::{ArraySchema, Name, RecordField as AvroRecordField, RecordSchema};
 use once_cell::sync::Lazy;
 use serde_json::{Number, Value as JsonValue};
 
diff --git a/crates/sail-iceberg/src/spec/metadata/table_metadata.rs b/crates/sail-iceberg/src/spec/metadata/table_metadata.rs
index 2f67862f2..ac4b752c9 100644
--- a/crates/sail-iceberg/src/spec/metadata/table_metadata.rs
+++ b/crates/sail-iceberg/src/spec/metadata/table_metadata.rs
@@ -27,7 +27,7 @@ use crate::spec::metadata::format::FormatVersion;
 use crate::spec::metadata::statistic_file::{PartitionStatisticsFile, StatisticsFile};
 use crate::spec::partition::PartitionSpec;
 use crate::spec::schema::Schema;
-use crate::spec::snapshots::{Snapshot, SnapshotReference, MAIN_BRANCH};
+use crate::spec::snapshots::{MAIN_BRANCH, Snapshot, SnapshotReference};
 use crate::spec::sort::SortOrder;
 
 /// Iceberg table metadata
diff --git a/crates/sail-iceberg/src/spec/partition/spec.rs b/crates/sail-iceberg/src/spec/partition/spec.rs
index 0665bb6da..ba96240bd 100644
--- a/crates/sail-iceberg/src/spec/partition/spec.rs
+++ b/crates/sail-iceberg/src/spec/partition/spec.rs
@@ -25,7 +25,6 @@ use crate::spec::schema::Schema;
 use crate::spec::transform::Transform;
 use crate::spec::types::{NestedField, StructType};
 
-#[expect(unused)]
 pub(crate) const UNPARTITIONED_LAST_ASSIGNED_ID: i32 = 999;
 pub(crate) const DEFAULT_PARTITION_SPEC_ID: i32 = 0;
 
@@ -167,6 +166,12 @@ impl PartitionSpec {
         self.fields.iter().map(|f| f.field_id).max()
     }
 
+    /// Return the Iceberg table metadata `last-partition-id` value for this spec.
+    pub fn last_assigned_field_id(&self) -> i32 {
+        self.highest_field_id()
+            .unwrap_or(UNPARTITIONED_LAST_ASSIGNED_ID)
+    }
+
     /// Check if the partition spec has sequential field ids starting from 1000.
     /// Required for spec version 1 in the reference implementation.
     pub fn has_sequential_ids(&self) -> bool {
diff --git a/crates/sail-iceberg/src/spec/schema/mod.rs b/crates/sail-iceberg/src/spec/schema/mod.rs
index d41506599..5346b5a70 100644
--- a/crates/sail-iceberg/src/spec/schema/mod.rs
+++ b/crates/sail-iceberg/src/spec/schema/mod.rs
@@ -30,7 +30,7 @@ mod visitor;
 
 pub use utils::*;
 
-use crate::schema::visitor::{visit_struct, SchemaVisitor};
+use crate::schema::visitor::{SchemaVisitor, visit_struct};
 use crate::{ListType, MapType};
 
 /// Type alias for schema id.
diff --git a/crates/sail-iceberg/src/spec/schema/visitor.rs b/crates/sail-iceberg/src/spec/schema/visitor.rs
index 56fa10232..d70a5d2f8 100644
--- a/crates/sail-iceberg/src/spec/schema/visitor.rs
+++ b/crates/sail-iceberg/src/spec/schema/visitor.rs
@@ -76,14 +76,14 @@ pub trait SchemaVisitor {
 
     /// Called after struct's fields visited.
     fn r#struct(&mut self, r#struct: &StructType, results: Vec)
-        -> Result;
+    -> Result;
 
     /// Called after list fields visited.
     fn list(&mut self, list: &ListType, value: Self::T) -> Result;
 
     /// Called after map's key and value fields visited.
     fn map(&mut self, map: &MapType, key_value: Self::T, value: Self::T)
-        -> Result;
+    -> Result;
 
     /// Called when see a primitive type.
     fn primitive(&mut self, p: &PrimitiveType) -> Result;
diff --git a/crates/sail-iceberg/src/spec/types/mod.rs b/crates/sail-iceberg/src/spec/types/mod.rs
index be248fd5d..f80bb17b0 100644
--- a/crates/sail-iceberg/src/spec/types/mod.rs
+++ b/crates/sail-iceberg/src/spec/types/mod.rs
@@ -26,7 +26,7 @@ use std::sync::{Arc, OnceLock};
 
 use ordered_float::OrderedFloat;
 use serde::de::{Error, IntoDeserializer, MapAccess, Visitor};
-use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
 use serde_json::Value as JsonValue;
 pub use values::*;
 
diff --git a/crates/sail-iceberg/src/spec/types/values.rs b/crates/sail-iceberg/src/spec/types/values.rs
index a6e7ea489..9c8d893d4 100644
--- a/crates/sail-iceberg/src/spec/types/values.rs
+++ b/crates/sail-iceberg/src/spec/types/values.rs
@@ -413,7 +413,7 @@ impl Literal {
             (expected, literal) => {
                 return Err(format!(
                     "Sail literal {literal:?} is not supported for Iceberg type {expected:?}"
-                ))
+                ));
             }
         })
     }
diff --git a/crates/sail-iceberg/src/spec/views/view_version.rs b/crates/sail-iceberg/src/spec/views/view_version.rs
index 0a770e404..c18f48c64 100644
--- a/crates/sail-iceberg/src/spec/views/view_version.rs
+++ b/crates/sail-iceberg/src/spec/views/view_version.rs
@@ -14,8 +14,8 @@ use std::collections::HashMap;
 
 use serde::{Deserialize, Serialize};
 
-use crate::spec::catalog::NamespaceIdent;
 use crate::spec::SchemaId;
+use crate::spec::catalog::NamespaceIdent;
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
diff --git a/crates/sail-iceberg/src/table/metadata_loader.rs b/crates/sail-iceberg/src/table/metadata_loader.rs
index 0fe3c3ea4..a058f67ae 100644
--- a/crates/sail-iceberg/src/table/metadata_loader.rs
+++ b/crates/sail-iceberg/src/table/metadata_loader.rs
@@ -14,12 +14,12 @@ use std::collections::HashMap;
 use std::io::{self, Read, Write};
 use std::sync::Arc;
 
-use datafusion::common::{plan_err, DataFusionError, Result};
+use datafusion::common::{DataFusionError, Result, plan_err};
+use flate2::Compression;
 use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
-use flate2::Compression;
-use object_store::path::Path as ObjectPath;
 use object_store::ObjectStoreExt;
+use object_store::path::Path as ObjectPath;
 use url::Url;
 
 const METADATA_COMPRESSION_PROPERTY: &str = "write.metadata.compression-codec";
@@ -165,23 +165,22 @@ pub async fn find_latest_metadata_file(
     let version_hint_path = base_path.clone().join("metadata").join("version-hint.text");
     let mut hinted_version: Option = None;
     let mut hinted_filename: Option = None;
-    if let Ok(version_hint_data) = object_store.get(&version_hint_path).await {
-        if let Ok(version_hint_bytes) = version_hint_data.bytes().await {
-            if let Ok(version_hint) = String::from_utf8(version_hint_bytes.to_vec()) {
-                let content = version_hint.trim();
-                if let Ok(version) = content.parse::() {
-                    log::trace!("Using numeric version hint: {}", version);
-                    hinted_version = Some(version);
-                } else {
-                    let fname = if parse_metadata_file_name(content).is_some() {
-                        content.to_string()
-                    } else {
-                        format!("{}.metadata.json", content)
-                    };
-                    log::trace!("Using filename version hint: {}", fname);
-                    hinted_filename = Some(fname);
-                }
-            }
+    if let Ok(version_hint_data) = object_store.get(&version_hint_path).await
+        && let Ok(version_hint_bytes) = version_hint_data.bytes().await
+        && let Ok(version_hint) = String::from_utf8(version_hint_bytes.to_vec())
+    {
+        let content = version_hint.trim();
+        if let Ok(version) = content.parse::() {
+            log::trace!("Using numeric version hint: {}", version);
+            hinted_version = Some(version);
+        } else {
+            let fname = if parse_metadata_file_name(content).is_some() {
+                content.to_string()
+            } else {
+                format!("{}.metadata.json", content)
+            };
+            log::trace!("Using filename version hint: {}", fname);
+            hinted_filename = Some(fname);
         }
     }
 
@@ -193,10 +192,10 @@ pub async fn find_latest_metadata_file(
     let metadata_files: Result, _> = objects
         .try_filter_map(|obj| async move {
             let path_str = obj.location.to_string();
-            if let Some(filename) = path_str.split('/').next_back() {
-                if let Some(metadata_file) = parse_metadata_file_name(filename) {
-                    return Ok(Some((metadata_file.version, path_str, obj.last_modified)));
-                }
+            if let Some(filename) = path_str.split('/').next_back()
+                && let Some(metadata_file) = parse_metadata_file_name(filename)
+            {
+                return Ok(Some((metadata_file.version, path_str, obj.last_modified)));
             }
             Ok(None)
         })
@@ -205,7 +204,7 @@ pub async fn find_latest_metadata_file(
 
     match metadata_files {
         Ok(mut files) => {
-            log::trace!("find_latest_metadata_file: found files: {:?}", &files);
+            log::trace!("find_latest_metadata_file: found files: {:?}", files);
             files.sort_by(|left, right| {
                 left.0
                     .cmp(&right.0)
@@ -213,33 +212,32 @@ pub async fn find_latest_metadata_file(
                     .then_with(|| left.1.cmp(&right.1))
             });
 
-            if let Some(fname) = hinted_filename {
-                if let Some((version, path, _)) =
+            if let Some(fname) = hinted_filename
+                && let Some((version, path, _)) =
                     files.iter().rev().find(|(_, p, _)| p.ends_with(&fname))
-                {
-                    log::trace!(
-                        "find_latest_metadata_file: selected by filename hint version {} path={}",
-                        version,
-                        &path
-                    );
-                    return Ok(path.clone());
-                }
-            } else if let Some(hint) = hinted_version {
-                if let Some((version, path, _)) = files.iter().rev().find(|(v, _, _)| *v == hint) {
-                    log::trace!(
-                        "find_latest_metadata_file: selected by numeric hint version {} path={}",
-                        version,
-                        &path
-                    );
-                    return Ok(path.clone());
-                }
+            {
+                log::trace!(
+                    "find_latest_metadata_file: selected by filename hint version {} path={}",
+                    version,
+                    path
+                );
+                return Ok(path.clone());
+            } else if let Some(hint) = hinted_version
+                && let Some((version, path, _)) = files.iter().rev().find(|(v, _, _)| *v == hint)
+            {
+                log::trace!(
+                    "find_latest_metadata_file: selected by numeric hint version {} path={}",
+                    version,
+                    path
+                );
+                return Ok(path.clone());
             }
 
             if let Some((version, latest_file, _)) = files.last() {
                 log::trace!(
                     "find_latest_metadata_file: selected version {} path={}",
                     version,
-                    &latest_file
+                    latest_file
                 );
                 Ok(latest_file.clone())
             } else {
@@ -258,13 +256,13 @@ mod tests {
     use std::io::{self, Write};
 
     use datafusion::common::Result;
-    use flate2::write::GzEncoder;
     use flate2::Compression;
+    use flate2::write::GzEncoder;
 
     use super::{
-        decode_metadata_file, encode_metadata_file, metadata_file_extension_from_properties,
-        metadata_location_to_object_path, parse_metadata_file_name, MetadataFileCodec,
-        MetadataFileName,
+        MetadataFileCodec, MetadataFileName, decode_metadata_file, encode_metadata_file,
+        metadata_file_extension_from_properties, metadata_location_to_object_path,
+        parse_metadata_file_name,
     };
 
     #[test]
diff --git a/crates/sail-iceberg/src/table/mod.rs b/crates/sail-iceberg/src/table/mod.rs
index 0457d26c0..803d2e6f1 100644
--- a/crates/sail-iceberg/src/table/mod.rs
+++ b/crates/sail-iceberg/src/table/mod.rs
@@ -21,7 +21,7 @@ use url::Url;
 use crate::datasource::provider::IcebergTableProvider;
 use crate::io::StoreContext;
 use crate::operations::Transaction;
-use crate::options::gen::IcebergReadOptions;
+use crate::options::r#gen::IcebergReadOptions;
 use crate::spec::snapshots::MAIN_BRANCH;
 use crate::spec::{PartitionSpec, Schema, Snapshot, TableMetadata};
 
@@ -44,7 +44,11 @@ impl Table {
         table_url: Url,
         metadata_location: Option,
     ) -> Result {
-        log::trace!("Loading Iceberg table: {}", table_url);
+        log::trace!(
+            "Loading Iceberg table: table_url={}, metadata_location={:?}",
+            table_url,
+            metadata_location,
+        );
         let object_store = ctx
             .runtime_env()
             .object_store_registry
@@ -125,10 +129,13 @@ impl Table {
 
     /// Create a Transaction anchored at the current snapshot, if one exists.
     pub fn new_transaction(&self) -> Option {
-        self.metadata
-            .current_snapshot()
-            .cloned()
-            .map(|snapshot| Transaction::new(self.table_url.to_string(), snapshot))
+        self.metadata.current_snapshot().cloned().map(|snapshot| {
+            Transaction::new(
+                self.table_url.to_string(),
+                snapshot,
+                self.metadata.last_sequence_number,
+            )
+        })
     }
 
     fn select_snapshot(&self, options: &IcebergReadOptions) -> Result<(Schema, Snapshot)> {
diff --git a/crates/sail-iceberg/src/table_format.rs b/crates/sail-iceberg/src/table_format.rs
index c54d827a3..e5002042f 100644
--- a/crates/sail-iceberg/src/table_format.rs
+++ b/crates/sail-iceberg/src/table_format.rs
@@ -10,31 +10,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
 use std::sync::Arc;
 
 use async_trait::async_trait;
 use bytes::Bytes;
 use datafusion::arrow::datatypes::{Field as ArrowField, Schema as ArrowSchema};
 use datafusion::catalog::{Session, TableProvider};
-use datafusion::common::{not_impl_err, plan_err, DataFusionError, Result};
+use datafusion::common::{
+    DataFusionError, Result, TableReference, ToDFSchema, not_impl_err, plan_err,
+};
 use datafusion::execution::SessionState;
-use datafusion::logical_expr::{LogicalPlan, TableSource};
+use datafusion::logical_expr::{LogicalPlan, TableScan, TableSource};
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_expr::expr::Sort;
 use datafusion_expr::{Expr, Extension, UserDefinedLogicalNodeCore};
 use educe::Educe;
+use log::warn;
 use object_store::ObjectStoreExt;
 use sail_common_datafusion::catalog::iceberg::is_iceberg_table_marker;
 use sail_common_datafusion::catalog::managed::metadata_location_value;
 use sail_common_datafusion::catalog::{
-    CatalogPartitionField, CommitAuthority, LakehouseExecutionContext, ScanAuthority,
+    CatalogPartitionField, CommitAuthority, LakehouseExecutionContext, LakehouseOperation,
+    ScanAuthority,
 };
 use sail_common_datafusion::datasource::{
-    create_sort_order, find_path_in_options, BucketBy, OptionLayer, PhysicalSinkMode, SinkInfo,
-    SinkMode, SourceInfo, TableFormat, TableFormatAlterTableOperation,
-    TableFormatCreateTableColumn, TableFormatCreateTableInfo, TableFormatCreateTableResult,
-    TableFormatRegistry,
+    BucketBy, DeleteInfo, OptionLayer, PhysicalSinkMode, SinkInfo, SinkMode, SourceInfo,
+    TableFormat, TableFormatAlterTableOperation, TableFormatCreateTableColumn,
+    TableFormatCreateTableInfo, TableFormatCreateTableResult, TableFormatRegistry,
+    create_sort_order, find_path_in_options,
 };
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_common_datafusion::variant::with_variant_extension_if_marked_storage;
@@ -42,22 +45,22 @@ use sail_data_source::options::ResolveOptions;
 use url::Url;
 
 use crate::datasource::provider::IcebergTableProvider;
-use crate::datasource::type_converter::{arrow_schema_to_iceberg, ICEBERG_ARROW_FIELD_DOC_KEY};
+use crate::datasource::type_converter::{ICEBERG_ARROW_FIELD_DOC_KEY, arrow_schema_to_iceberg};
 use crate::io::StoreContext;
 use crate::logical::IcebergTableSource;
 use crate::operations::bootstrap::{
-    bootstrap_empty_table_metadata, replace_empty_table_metadata, NewTableMetadataStyle,
+    NewTableMetadataStyle, bootstrap_empty_table_metadata, replace_empty_table_metadata,
 };
-use crate::options::gen::{IcebergReadOptions, IcebergWriteOptions};
-use crate::physical_plan::plan_builder::{IcebergPlanBuilder, IcebergTableConfig};
+use crate::options::r#gen::{IcebergReadOptions, IcebergWriteOptions};
 use crate::physical_plan::IcebergWriterExecOptions;
+use crate::physical_plan::plan_builder::{IcebergPlanBuilder, IcebergTableConfig};
 use crate::schema_evolution::SchemaEvolver;
 use crate::spec::{MetadataLog, PartitionSpec, Schema, Snapshot, TableMetadata};
 use crate::table::metadata_loader::{
     encode_metadata_file, load_metadata_file_bytes, metadata_file_extension_from_properties,
     metadata_file_version_from_path, metadata_location_to_object_path_string,
 };
-use crate::table::{find_latest_metadata_file, Table};
+use crate::table::{Table, find_latest_metadata_file};
 use crate::utils::metadata::metadata_files_for_version;
 use crate::utils::partition_transform::{
     catalog_partition_field_from_iceberg, format_partition_expr, format_partition_exprs,
@@ -143,6 +146,71 @@ impl TableFormat for IcebergTableFormat {
         }))
     }
 
+    async fn create_deleter(&self, ctx: &dyn Session, info: DeleteInfo) -> Result {
+        let DeleteInfo {
+            table_name,
+            path,
+            condition,
+            lakehouse_table,
+            options,
+        } = info;
+
+        let read_lakehouse_table = lakehouse_table
+            .as_ref()
+            .map(|context| context.for_operation(LakehouseOperation::Read));
+        let source_info = SourceInfo {
+            paths: vec![path.clone()],
+            lakehouse_table: read_lakehouse_table,
+            schema: None,
+            constraints: Default::default(),
+            partition_by: vec![],
+            bucket_by: None,
+            sort_order: vec![],
+            options: options.clone(),
+            // TODO: Thread resolver session case-sensitivity into TableFormat::create_deleter.
+            read_case_sensitive: true,
+        };
+        let provider = build_iceberg_provider(ctx, source_info).await?;
+        let expected_snapshot_id = Some(
+            provider
+                .current_snapshot()
+                .map(|snapshot| snapshot.snapshot_id()),
+        );
+        let table_source: Arc = Arc::new(IcebergTableSource::new(provider));
+        let raw_input_schema = table_source.schema().to_dfschema_ref()?;
+        let target_scan = LogicalPlan::TableScan(TableScan::try_new(
+            table_reference_from_parts(&table_name),
+            table_source,
+            None,
+            vec![],
+            None,
+        )?);
+
+        let write_node = sail_logical_plan::merge::RowLevelWriteNode::new_delete(
+            Arc::new(target_scan),
+            raw_input_schema,
+            condition,
+            self.name().to_string(),
+            path,
+            table_name,
+            options,
+            lakehouse_table,
+        )
+        .with_expected_snapshot_id(expected_snapshot_id);
+
+        Ok(LogicalPlan::Extension(Extension {
+            node: Arc::new(write_node),
+        }))
+    }
+
+    async fn create_merger(
+        &self,
+        _ctx: &dyn Session,
+        info: sail_common_datafusion::datasource::MergeInfo,
+    ) -> Result {
+        crate::logical::merge::expand_merge_node(info)
+    }
+
     async fn create_table_metadata(
         &self,
         runtime_env: Arc,
@@ -442,27 +510,28 @@ pub(crate) async fn plan_iceberg_write(
         None
     };
 
-    if let Some(existing_partitions) = &existing_partition_columns {
-        if !partition_by.is_empty() && partition_by != *existing_partitions {
-            match mode {
-                PhysicalSinkMode::Append => {
-                    return plan_err!(
-                        "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \
+    if let Some(existing_partitions) = &existing_partition_columns
+        && !partition_by.is_empty()
+        && partition_by != *existing_partitions
+    {
+        match mode {
+            PhysicalSinkMode::Append => {
+                return plan_err!(
+                    "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \
                         Cannot change partitioning on append.",
-                        format_partition_exprs(existing_partitions),
-                        format_partition_exprs(&partition_by)
-                    );
-                }
-                PhysicalSinkMode::Overwrite if !iceberg_options.overwrite_schema => {
-                    return plan_err!(
-                        "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \
+                    format_partition_exprs(existing_partitions),
+                    format_partition_exprs(&partition_by)
+                );
+            }
+            PhysicalSinkMode::Overwrite if !iceberg_options.overwrite_schema => {
+                return plan_err!(
+                    "Partition column mismatch. Table is partitioned by {:?}, but write specified {:?}. \
                         Set overwriteSchema=true to change partitioning.",
-                        format_partition_exprs(existing_partitions),
-                        format_partition_exprs(&partition_by)
-                    );
-                }
-                _ => {}
+                    format_partition_exprs(existing_partitions),
+                    format_partition_exprs(&partition_by)
+                );
             }
+            _ => {}
         }
     }
 
@@ -542,10 +611,11 @@ impl IcebergTableFormat {
                 continue;
             }
 
+            let previous_metadata_timestamp_ms = table_meta.last_updated_ms;
             let timestamp_ms = crate::utils::timestamp::monotonic_timestamp_ms();
             table_meta.last_updated_ms = timestamp_ms;
             table_meta.metadata_log.push(MetadataLog {
-                timestamp_ms,
+                timestamp_ms: previous_metadata_timestamp_ms,
                 metadata_file: latest_meta.clone(),
             });
 
@@ -619,10 +689,7 @@ impl IcebergTableFormat {
         }
     }
 
-    // TODO: Implement row-level DELETE/UPDATE/MERGE for this format. Expanded
-    // inputs should consume Sail row intent tags to decide which rows rewrite
-    // data files and which rows produce low-level delete artifacts, then strip
-    // all internal metadata before writing user data.
+    // TODO: Add row-level UPDATE and configurable COW/MOR strategy selection.
 }
 
 /// Create an Iceberg table provider for reading.
@@ -708,8 +775,8 @@ fn validate_iceberg_lakehouse_storage_access(
         .is_some_and(|session| session.remote_signing_enabled)
     {
         // TODO: Wire REST remote signing into Iceberg FileIO/object-store access.
-        return not_impl_err!(
-            "Iceberg REST catalog table {} requires remote signing, which is not implemented yet",
+        warn!(
+            "Iceberg REST catalog table {} advertises remote signing, which is not implemented yet",
             context.catalog_table().join(".")
         );
     }
@@ -719,8 +786,8 @@ fn validate_iceberg_lakehouse_storage_access(
         .is_some_and(|session| session.storage_credential_count > 0)
     {
         // TODO: Apply REST vended credentials to operation-scoped storage access.
-        return not_impl_err!(
-            "Iceberg REST catalog table {} requires vended storage credentials, which is not implemented yet",
+        warn!(
+            "Iceberg REST catalog table {} advertises vended storage credentials, which is not implemented yet",
             context.catalog_table().join(".")
         );
     }
@@ -755,7 +822,7 @@ impl IcebergTableFormat {
         let path = &paths[0];
         let mut table_url = match crate::utils::parse_absolute_url(path) {
             Some(url) => url,
-            _ => file_url_from_absolute_path(path).ok_or_else(|| {
+            _ => crate::utils::file_url_from_absolute_path(path).ok_or_else(|| {
                 DataFusionError::Plan(format!(
                     "Iceberg table location must be an absolute path or URL: {path}"
                 ))
@@ -768,7 +835,9 @@ impl IcebergTableFormat {
         Ok(table_url)
     }
 
-    fn partition_columns_from_metadata(table: &Table) -> Result> {
+    pub(crate) fn partition_columns_from_metadata(
+        table: &Table,
+    ) -> Result> {
         partition_columns_from_table_metadata(table.metadata())
     }
 }
@@ -808,6 +877,26 @@ fn partition_columns_from_table_metadata(
     Ok(columns)
 }
 
+fn table_reference_from_parts(parts: &[String]) -> TableReference {
+    match parts {
+        [table] => TableReference::Bare {
+            table: table.as_str().into(),
+        },
+        [schema, table] => TableReference::Partial {
+            schema: schema.as_str().into(),
+            table: table.as_str().into(),
+        },
+        [catalog, schema, table] => TableReference::Full {
+            catalog: catalog.as_str().into(),
+            schema: schema.as_str().into(),
+            table: table.as_str().into(),
+        },
+        _ => TableReference::Bare {
+            table: parts.join(".").into(),
+        },
+    }
+}
+
 fn create_table_arrow_schema(columns: Vec) -> Result {
     let fields = columns
         .into_iter()
@@ -896,27 +985,6 @@ fn next_partition_spec_id(metadata: &TableMetadata) -> i32 {
         + 1
 }
 
-fn file_url_from_absolute_path(path: &str) -> Option {
-    if Path::new(path).is_absolute() {
-        return Url::from_file_path(path).ok();
-    }
-    windows_drive_path_to_file_url(path)
-}
-
-fn windows_drive_path_to_file_url(path: &str) -> Option {
-    let bytes = path.as_bytes();
-    if bytes.len() < 3
-        || !bytes[0].is_ascii_alphabetic()
-        || bytes[1] != b':'
-        || !matches!(bytes[2], b'/' | b'\\')
-    {
-        return None;
-    }
-
-    let path = path.replace('\\', "/");
-    Url::parse(&format!("file:///{path}")).ok()
-}
-
 pub(crate) fn table_metadata_location(table_url: &Url, metadata_file: &str) -> Result {
     if crate::utils::parse_absolute_url(metadata_file).is_some() {
         return Ok(metadata_file.to_string());
@@ -936,12 +1004,11 @@ fn relative_metadata_file(table_url: &Url, metadata_file: &str) -> Result>) -> Self {
-        if let Some(p) = props {
-            if let Some(v) = p.get(WRITE_RELATIVE_PROP) {
-                if v.eq_ignore_ascii_case("true") || v == "1" {
-                    return WritePathMode::Relative;
-                }
-            }
+        if let Some(p) = props
+            && let Some(v) = p.get(WRITE_RELATIVE_PROP)
+            && (v.eq_ignore_ascii_case("true") || v == "1")
+        {
+            return WritePathMode::Relative;
         }
         WritePathMode::Absolute
     }
@@ -55,6 +55,27 @@ pub fn parse_absolute_url(raw: &str) -> Option {
     Url::parse(raw).ok().filter(|url| url.scheme().len() > 1)
 }
 
+pub fn file_url_from_absolute_path(path: &str) -> Option {
+    if Path::new(path).is_absolute() {
+        return Url::from_file_path(path).ok();
+    }
+    windows_drive_path_to_file_url(path)
+}
+
+fn windows_drive_path_to_file_url(path: &str) -> Option {
+    let bytes = path.as_bytes();
+    if bytes.len() < 3
+        || !bytes[0].is_ascii_alphabetic()
+        || bytes[1] != b':'
+        || !matches!(bytes[2], b'/' | b'\\')
+    {
+        return None;
+    }
+
+    let path = path.replace('\\', "/");
+    Url::parse(&format!("file:///{path}")).ok()
+}
+
 pub fn url_to_object_path(url: &Url) -> Result {
     let is_file = url.scheme() == "file";
     let p = if is_file {
diff --git a/crates/sail-iceberg/src/utils/partition_transform.rs b/crates/sail-iceberg/src/utils/partition_transform.rs
index 38c576250..0dd17a8ed 100644
--- a/crates/sail-iceberg/src/utils/partition_transform.rs
+++ b/crates/sail-iceberg/src/utils/partition_transform.rs
@@ -57,7 +57,7 @@ pub fn catalog_partition_field_from_iceberg(
         other => {
             return Err(format!(
                 "unsupported Iceberg partition transform '{other}' for column '{source_column}'"
-            ))
+            ));
         }
     };
     Ok(CatalogPartitionField {
diff --git a/crates/sail-iceberg/src/utils/timestamp.rs b/crates/sail-iceberg/src/utils/timestamp.rs
index a4f489daf..94c3c8f0b 100644
--- a/crates/sail-iceberg/src/utils/timestamp.rs
+++ b/crates/sail-iceberg/src/utils/timestamp.rs
@@ -16,17 +16,16 @@ static LAST_TIMESTAMP_MS: AtomicI64 = AtomicI64::new(0);
 
 /// Generate a monotonically increasing timestamp in milliseconds.
 pub fn monotonic_timestamp_ms() -> i64 {
-    loop {
-        let now = chrono::Utc::now().timestamp_millis();
-        let last = LAST_TIMESTAMP_MS.load(Ordering::Relaxed);
-
-        let next = if now > last { now } else { last + 1 };
+    let mut timestamp = 0;
+    LAST_TIMESTAMP_MS.update(Ordering::SeqCst, Ordering::Relaxed, |last| {
+        timestamp = next_monotonic_timestamp(chrono::Utc::now().timestamp_millis(), last);
+        timestamp
+    });
+    timestamp
+}
 
-        match LAST_TIMESTAMP_MS.compare_exchange(last, next, Ordering::SeqCst, Ordering::Relaxed) {
-            Ok(_) => return next,
-            Err(_) => continue,
-        }
-    }
+fn next_monotonic_timestamp(now: i64, last: i64) -> i64 {
+    if now > last { now } else { last + 1 }
 }
 
 #[cfg(test)]
diff --git a/crates/sail-logical-optimizer/src/lateral_join.rs b/crates/sail-logical-optimizer/src/lateral_join.rs
index 7fe842a05..f340027ed 100644
--- a/crates/sail-logical-optimizer/src/lateral_join.rs
+++ b/crates/sail-logical-optimizer/src/lateral_join.rs
@@ -1,8 +1,8 @@
 use std::sync::Arc;
 
 use datafusion::optimizer::{ApplyOrder, OptimizerConfig, OptimizerRule};
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
 use datafusion_common::Result;
+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
 use datafusion_expr::{Expr, JoinType, LogicalPlan, LogicalPlanBuilder};
 
 #[derive(Debug, Default)]
diff --git a/crates/sail-logical-optimizer/src/resolve_lambda_variables.rs b/crates/sail-logical-optimizer/src/resolve_lambda_variables.rs
index b448ea56e..b9b72790b 100644
--- a/crates/sail-logical-optimizer/src/resolve_lambda_variables.rs
+++ b/crates/sail-logical-optimizer/src/resolve_lambda_variables.rs
@@ -1,7 +1,7 @@
 use datafusion::optimizer::{AnalyzerRule, ApplyOrder, OptimizerConfig, OptimizerRule};
+use datafusion_common::Result;
 use datafusion_common::config::ConfigOptions;
 use datafusion_common::tree_node::Transformed;
-use datafusion_common::Result;
 use datafusion_expr::LogicalPlan;
 
 #[derive(Debug, Default)]
diff --git a/crates/sail-logical-plan/src/barrier.rs b/crates/sail-logical-plan/src/barrier.rs
index 3a1f9bbd3..5574c68c9 100644
--- a/crates/sail-logical-plan/src/barrier.rs
+++ b/crates/sail-logical-plan/src/barrier.rs
@@ -1,7 +1,7 @@
 use std::fmt::Formatter;
 use std::sync::Arc;
 
-use datafusion_common::{plan_err, DFSchemaRef};
+use datafusion_common::{DFSchemaRef, plan_err};
 use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
 use sail_common_datafusion::utils::items::ItemTaker;
 
diff --git a/crates/sail-logical-plan/src/check_constraints.rs b/crates/sail-logical-plan/src/check_constraints.rs
index 24d3e29f3..190bf5b10 100644
--- a/crates/sail-logical-plan/src/check_constraints.rs
+++ b/crates/sail-logical-plan/src/check_constraints.rs
@@ -1,5 +1,5 @@
 use datafusion_common::Result;
-use datafusion_expr::{lit, when, Expr, LogicalPlan, LogicalPlanBuilder, ScalarUDF};
+use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, ScalarUDF, lit, when};
 pub use sail_common_datafusion::datasource::{DeltaCheckConstraintExpr, DeltaConstraintViolation};
 use sail_function::scalar::misc::raise_error::RaiseError;
 
diff --git a/crates/sail-logical-plan/src/lib.rs b/crates/sail-logical-plan/src/lib.rs
index fa6d6a904..26cf741e1 100644
--- a/crates/sail-logical-plan/src/lib.rs
+++ b/crates/sail-logical-plan/src/lib.rs
@@ -4,6 +4,7 @@ pub mod map_partitions;
 pub mod merge;
 pub mod monotonic_id;
 pub mod range;
+pub mod remote_checkpoint;
 pub mod repartition;
 pub mod schema_pivot;
 pub mod show_string;
diff --git a/crates/sail-logical-plan/src/merge.rs b/crates/sail-logical-plan/src/merge.rs
index a97638292..17e9c94b1 100644
--- a/crates/sail-logical-plan/src/merge.rs
+++ b/crates/sail-logical-plan/src/merge.rs
@@ -1,4 +1,4 @@
-use std::collections::{HashMap, VecDeque};
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::fmt::Formatter;
 use std::hash::Hash;
 use std::sync::Arc;
@@ -7,8 +7,8 @@ use datafusion::functions_aggregate::count::count_udaf;
 use datafusion_common::tree_node::{Transformed, TreeNode};
 use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
 use datafusion_common::{
-    plan_err, Column, DFSchema, DFSchemaRef, DataFusionError, Dependency, NullEquality, Result,
-    ScalarValue, TableReference,
+    Column, DFSchema, DFSchemaRef, DataFusionError, Dependency, NullEquality, Result, ScalarValue,
+    TableReference, plan_err,
 };
 use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams, Case};
 use datafusion_expr::expr_fn::not;
@@ -17,8 +17,8 @@ use datafusion_expr::logical_plan::{
 };
 use datafusion_expr::utils::expr_to_columns;
 use datafusion_expr::{
-    col, lit, when, BinaryExpr, Expr, Join, JoinConstraint, JoinType, LogicalPlan, Operator,
-    ScalarUDF, UserDefinedLogicalNodeCore,
+    BinaryExpr, Expr, Join, JoinConstraint, JoinType, LogicalPlan, Operator, ScalarUDF,
+    UserDefinedLogicalNodeCore, cast, col, lit, when,
 };
 use educe::Educe;
 use log::trace;
@@ -35,10 +35,10 @@ pub const TARGET_ROW_ID_COLUMN: &str = "__sail_merge_target_row_id";
 
 use sail_common_datafusion::catalog::LakehouseExecutionContext;
 pub use sail_common_datafusion::datasource::{
-    DeltaCheckConstraintExpr, MergeAssignment, MergeInfo, MergeIntoOptions, MergeMatchedAction,
-    MergeMatchedClause, MergeNotMatchedBySourceAction, MergeNotMatchedBySourceClause,
-    MergeNotMatchedByTargetAction, MergeNotMatchedByTargetClause, MergeTargetInfo,
-    MERGE_SOURCE_METRIC_COLUMN, OPERATION_COLUMN,
+    DeltaCheckConstraintExpr, MERGE_SOURCE_METRIC_COLUMN, MergeAssignment, MergeInfo,
+    MergeIntoOptions, MergeMatchedAction, MergeMatchedClause, MergeNotMatchedBySourceAction,
+    MergeNotMatchedBySourceClause, MergeNotMatchedByTargetAction, MergeNotMatchedByTargetClause,
+    MergeTargetInfo, OPERATION_COLUMN,
 };
 use sail_common_datafusion::datasource::{OptionLayer, RowLevelOperationType};
 
@@ -150,8 +150,8 @@ pub struct RowLevelWriteNode {
     write_plan: Option>,
     /// Plan yielding touched file paths (MERGE targeted rewrite).
     touched_files_plan: Option>,
-    /// Plan yielding file path and file-local row index for MERGE rows deleted via DVs.
-    deletion_vector_plan: Option>,
+    /// Plan yielding file path and file-local row index for MERGE target rows to delete.
+    row_index_delete_plan: Option>,
     /// Condition for DELETE/UPDATE (passed through to physical planner).
     #[educe(PartialOrd(ignore))]
     condition: Option,
@@ -163,6 +163,9 @@ pub struct RowLevelWriteNode {
     target_partition_by: Vec,
     target_options: Vec,
     target_lakehouse_table: Option,
+    /// `Some` marks a row-level write whose target scan must still match at commit time.
+    /// The inner value is `None` when the target had no snapshot when it was read.
+    expected_snapshot_id: Option>,
     with_schema_evolution: bool,
     #[educe(PartialOrd(ignore))]
     schema: DFSchemaRef,
@@ -176,7 +179,7 @@ impl RowLevelWriteNode {
         raw_input_schema: DFSchemaRef,
         write_plan: Arc,
         touched_files_plan: Arc,
-        deletion_vector_plan: Option>,
+        row_index_delete_plan: Option>,
         options: MergeIntoOptions,
         schema: DFSchemaRef,
     ) -> Self {
@@ -188,19 +191,25 @@ impl RowLevelWriteNode {
             target_partition_by: options.target.partition_by.clone(),
             target_options: options.target.options.clone(),
             target_lakehouse_table: options.target.lakehouse_table.clone(),
+            expected_snapshot_id: None,
             with_schema_evolution: options.with_schema_evolution,
             raw_target,
             raw_source: Some(raw_source),
             raw_input_schema,
             write_plan: Some(write_plan),
             touched_files_plan: Some(touched_files_plan),
-            deletion_vector_plan,
+            row_index_delete_plan,
             condition: None,
             merge_options: Some(options),
             schema,
         }
     }
 
+    pub fn with_expected_snapshot_id(mut self, expected_snapshot_id: Option>) -> Self {
+        self.expected_snapshot_id = expected_snapshot_id;
+        self
+    }
+
     /// Create a DELETE write node carrying the condition for the physical planner.
     pub fn new_delete(
         raw_target: Arc,
@@ -219,7 +228,7 @@ impl RowLevelWriteNode {
             raw_input_schema,
             write_plan: None,
             touched_files_plan: None,
-            deletion_vector_plan: None,
+            row_index_delete_plan: None,
             condition,
             merge_options: None,
             target_format: format,
@@ -228,6 +237,7 @@ impl RowLevelWriteNode {
             target_partition_by: Vec::new(),
             target_options: options,
             target_lakehouse_table: lakehouse_table,
+            expected_snapshot_id: None,
             with_schema_evolution: false,
             schema: Arc::new(DFSchema::empty()),
         }
@@ -261,8 +271,8 @@ impl RowLevelWriteNode {
         self.touched_files_plan.as_ref()
     }
 
-    pub fn deletion_vector_plan(&self) -> Option<&Arc> {
-        self.deletion_vector_plan.as_ref()
+    pub fn row_index_delete_plan(&self) -> Option<&Arc> {
+        self.row_index_delete_plan.as_ref()
     }
 
     pub fn condition(&self) -> Option<&ExprWithSource> {
@@ -293,6 +303,10 @@ impl RowLevelWriteNode {
         self.target_lakehouse_table.as_ref()
     }
 
+    pub fn expected_snapshot_id(&self) -> Option> {
+        self.expected_snapshot_id
+    }
+
     pub fn with_schema_evolution(&self) -> bool {
         self.with_schema_evolution
     }
@@ -311,8 +325,8 @@ impl UserDefinedLogicalNodeCore for RowLevelWriteNode {
         if let Some(tp) = &self.touched_files_plan {
             inputs.push(tp.as_ref());
         }
-        if let Some(dvp) = &self.deletion_vector_plan {
-            inputs.push(dvp.as_ref());
+        if let Some(plan) = &self.row_index_delete_plan {
+            inputs.push(plan.as_ref());
         }
         inputs
     }
@@ -381,10 +395,10 @@ impl UserDefinedLogicalNodeCore for RowLevelWriteNode {
         } else {
             None
         };
-        let deletion_vector_plan = if self.deletion_vector_plan.is_some() {
+        let row_index_delete_plan = if self.row_index_delete_plan.is_some() {
             Some(Arc::new(iter.next().ok_or_else(|| {
                 DataFusionError::Internal(
-                    "RowLevelWriteNode: missing deletion_vector_plan input".into(),
+                    "RowLevelWriteNode: missing row_index_delete_plan input".into(),
                 )
             })?))
         } else {
@@ -397,7 +411,7 @@ impl UserDefinedLogicalNodeCore for RowLevelWriteNode {
             raw_input_schema: self.raw_input_schema.clone(),
             write_plan,
             touched_files_plan,
-            deletion_vector_plan,
+            row_index_delete_plan,
             condition: self.condition.clone(),
             merge_options: self.merge_options.clone(),
             target_format: self.target_format.clone(),
@@ -406,6 +420,7 @@ impl UserDefinedLogicalNodeCore for RowLevelWriteNode {
             target_partition_by: self.target_partition_by.clone(),
             target_options: self.target_options.clone(),
             target_lakehouse_table: self.target_lakehouse_table.clone(),
+            expected_snapshot_id: self.expected_snapshot_id,
             with_schema_evolution: self.with_schema_evolution,
             schema: self.schema.clone(),
         })
@@ -420,15 +435,78 @@ impl UserDefinedLogicalNodeCore for RowLevelWriteNode {
 pub struct MergeExpansion {
     pub write_plan: LogicalPlan,
     pub touched_files_plan: LogicalPlan,
-    pub deletion_vector_plan: Option,
+    pub row_index_delete_plan: Option,
     pub output_schema: DFSchemaRef,
     pub options: MergeIntoOptions,
 }
 
+fn merge_name_key(name: &str, case_sensitive: bool) -> String {
+    if case_sensitive {
+        name.to_string()
+    } else {
+        name.to_ascii_lowercase()
+    }
+}
+
+fn merge_names_equal(left: &str, right: &str, case_sensitive: bool) -> bool {
+    if case_sensitive {
+        left == right
+    } else {
+        left.eq_ignore_ascii_case(right)
+    }
+}
+
+fn collision_free_source_alias(
+    source_name: &str,
+    occupied_names: &mut HashSet,
+    case_sensitive: bool,
+) -> String {
+    let base = format!("__sail_src_{source_name}");
+    let mut alias = base.clone();
+    let mut suffix = 0usize;
+    while occupied_names.contains(&merge_name_key(&alias, case_sensitive)) {
+        suffix += 1;
+        alias = format!("{base}_{suffix}");
+    }
+    occupied_names.insert(merge_name_key(&alias, case_sensitive));
+    alias
+}
+
+pub fn validate_merge_internal_columns(info: &MergeInfo, format_columns: &[&str]) -> Result<()> {
+    let target_schema = info.target.schema();
+    let reserved_columns = [
+        TARGET_PRESENT_COLUMN,
+        SOURCE_PRESENT_COLUMN,
+        TARGET_ROW_ID_COLUMN,
+        OPERATION_COLUMN,
+        MERGE_SOURCE_METRIC_COLUMN,
+    ]
+    .into_iter()
+    .chain(format_columns.iter().copied());
+
+    for reserved_column in reserved_columns {
+        if info
+            .options
+            .resolved_target_field_names
+            .iter()
+            .any(|name| merge_names_equal(name, reserved_column, info.options.case_sensitive))
+            || target_schema.fields().iter().any(|field| {
+                merge_names_equal(field.name(), reserved_column, info.options.case_sensitive)
+            })
+        {
+            return plan_err!(
+                "MERGE target column '{reserved_column}' uses a reserved internal MERGE column name"
+            );
+        }
+    }
+    Ok(())
+}
+
 pub fn expand_merge(
     info: MergeInfo,
     path_column: &str,
     row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
 ) -> Result {
     let target_plan = info.target.as_ref().clone();
     let source_plan = info.source.as_ref().clone();
@@ -492,8 +570,8 @@ pub fn expand_merge(
                 .collect()
         })
     };
-    trace!("resolved target names: {:?}", &desired_target_names);
-    trace!("resolved source names: {:?}", &desired_source_names);
+    trace!("resolved target names: {:?}", desired_target_names);
+    trace!("resolved source names: {:?}", desired_source_names);
 
     let _target_relation = options.target_alias.as_ref().map(|a| TableReference::Bare {
         table: a.clone().into(),
@@ -510,7 +588,7 @@ pub fn expand_merge(
         .collect();
     trace!(
         "target scan fields pre-projection: {:?}",
-        &target_scan_fields
+        target_scan_fields
     );
 
     let mut target_proj_exprs: Vec = target_plan
@@ -544,6 +622,21 @@ pub fn expand_merge(
             );
         }
     }
+    for metadata_column in row_delete_metadata_columns {
+        let already_present = target_proj_exprs
+            .iter()
+            .any(|expr| matches!(expr, Expr::Alias(alias) if alias.name == *metadata_column));
+        if !already_present
+            && target_plan
+                .schema()
+                .has_column_with_unqualified_name(metadata_column)
+        {
+            target_proj_exprs.push(
+                Expr::Column(Column::from_name((*metadata_column).to_string()))
+                    .alias((*metadata_column).to_string()),
+            );
+        }
+    }
 
     trace!(
         "target projection expr names: {:?}",
@@ -568,7 +661,7 @@ pub fn expand_merge(
         });
     }
 
-    // To avoid duplicate unqualified names after JOIN, rename source columns with a stable prefix.
+    // To avoid duplicate unqualified names after JOIN, assign collision-free source aliases.
     let target_input_len = options.resolved_target_schema.fields().len();
     let mut target_rename_map: HashMap = HashMap::new();
     for (idx, desired) in desired_target_names
@@ -590,19 +683,36 @@ pub fn expand_merge(
     if let Some(row_index_column) = row_index_column {
         target_rename_map.insert(row_index_column.to_string(), row_index_column.to_string());
     }
+    for metadata_column in row_delete_metadata_columns {
+        target_rename_map.insert(
+            (*metadata_column).to_string(),
+            (*metadata_column).to_string(),
+        );
+    }
     // keep row id stable if present
     target_rename_map.insert(
         TARGET_ROW_ID_COLUMN.to_string(),
         TARGET_ROW_ID_COLUMN.to_string(),
     );
 
+    let mut occupied_names = target_plan
+        .schema()
+        .fields()
+        .iter()
+        .map(|field| merge_name_key(field.name(), options.case_sensitive))
+        .collect::>();
     let mut source_rename_map: HashMap = HashMap::new();
+    options.source_column_aliases.clear();
     let target_input_len = options.resolved_target_schema.fields().len();
     for (idx, desired) in desired_source_names.iter().enumerate() {
-        let prefixed = format!("__sail_src_{desired}");
-        source_rename_map.insert(desired.clone(), prefixed.clone());
+        let source_alias =
+            collision_free_source_alias(desired, &mut occupied_names, options.case_sensitive);
+        options
+            .source_column_aliases
+            .push((desired.clone(), source_alias.clone()));
+        source_rename_map.insert(desired.clone(), source_alias.clone());
         if let Some(field) = merge_schema.fields().get(target_input_len + idx) {
-            source_rename_map.insert(field.name().clone(), prefixed.clone());
+            source_rename_map.insert(field.name().clone(), source_alias);
         }
     }
 
@@ -672,6 +782,11 @@ pub fn expand_merge(
         .iter()
         .map(|(name, expr)| Ok((name.clone(), rewrite(expr.clone())?)))
         .collect::>>()?;
+    options.default_column_exprs = options
+        .default_column_exprs
+        .iter()
+        .map(|(name, expr)| Ok((name.clone(), rewrite(expr.clone())?)))
+        .collect::>>()?;
     options.check_constraint_exprs = options
         .check_constraint_exprs
         .iter()
@@ -686,11 +801,11 @@ pub fn expand_merge(
         .collect::>>()?;
     trace!(
         "expand_merge options after rewrite - join_key_pairs: {:?}, matched_clauses: {:?}, not_matched_by_source_clauses: {:?}, not_matched_by_target_clauses: {:?}, on_condition: {:?}",
-        &options.join_key_pairs,
-        &options.matched_clauses,
-        &options.not_matched_by_source_clauses,
-        &options.not_matched_by_target_clauses,
-        &options.on_condition
+        options.join_key_pairs,
+        options.matched_clauses,
+        options.not_matched_by_source_clauses,
+        options.not_matched_by_target_clauses,
+        options.on_condition
     );
 
     // Detect insert-only MERGE that can use fast append (no touched files).
@@ -724,9 +839,9 @@ pub fn expand_merge(
         let insert_projection_exprs = build_insert_only_projection(
             &options,
             target_schema,
-            source_schema,
             path_column,
             row_index_column,
+            row_delete_metadata_columns,
             insert_operation,
         )?;
         let insert_projected = LogicalPlanBuilder::from(insert_rows)
@@ -746,8 +861,12 @@ pub fn expand_merge(
             false,
         )?;
         let noop_rows = LogicalPlan::Join(noop_rows);
-        let noop_projection_exprs =
-            build_insert_only_noop_projection(target_schema, path_column, row_index_column)?;
+        let noop_projection_exprs = build_insert_only_noop_projection(
+            target_schema,
+            path_column,
+            row_index_column,
+            row_delete_metadata_columns,
+        )?;
         let noop_projected = LogicalPlanBuilder::from(noop_rows)
             .project(noop_projection_exprs)?
             .build()?;
@@ -768,7 +887,7 @@ pub fn expand_merge(
         return Ok(MergeExpansion {
             write_plan: projected,
             touched_files_plan: touched_plan,
-            deletion_vector_plan: None,
+            row_index_delete_plan: None,
             output_schema: command_schema,
             options,
         });
@@ -781,6 +900,7 @@ pub fn expand_merge(
         should_check_cardinality,
         path_column,
         row_index_column,
+        row_delete_metadata_columns,
     )
 }
 
@@ -792,6 +912,7 @@ fn build_default_merge_expansion(
     should_check_cardinality: bool,
     path_column: &str,
     row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
 ) -> Result {
     let target_schema = target_plan.schema();
     let source_schema = source_plan.schema();
@@ -847,7 +968,7 @@ fn build_default_merge_expansion(
     let not_matched_by_source_pred = target_present.clone().and(not(source_present.clone()));
     let not_matched_by_target_pred = not(target_present.clone()).and(source_present.clone());
 
-    let mut delete_pred: Option = None;
+    let mut row_delete_pred: Option = None;
     let mut insert_pred: Option = None;
 
     for clause in &options.matched_clauses {
@@ -856,10 +977,11 @@ fn build_default_merge_expansion(
             pred = pred.and(cond.expr.clone());
         }
         match clause.action {
-            MergeMatchedAction::Delete => {
-                delete_pred = or_pred(delete_pred, pred);
+            MergeMatchedAction::Delete
+            | MergeMatchedAction::UpdateAll
+            | MergeMatchedAction::UpdateSet(_) => {
+                row_delete_pred = or_pred(row_delete_pred, pred);
             }
-            MergeMatchedAction::UpdateAll | MergeMatchedAction::UpdateSet(_) => {}
         }
     }
 
@@ -869,10 +991,9 @@ fn build_default_merge_expansion(
             pred = pred.and(cond.expr.clone());
         }
         match clause.action {
-            MergeNotMatchedBySourceAction::Delete => {
-                delete_pred = or_pred(delete_pred, pred);
+            MergeNotMatchedBySourceAction::Delete | MergeNotMatchedBySourceAction::UpdateSet(_) => {
+                row_delete_pred = or_pred(row_delete_pred, pred);
             }
-            MergeNotMatchedBySourceAction::UpdateSet(_) => {}
         }
     }
 
@@ -889,7 +1010,7 @@ fn build_default_merge_expansion(
         }
     }
 
-    let delete_expr = delete_pred.unwrap_or_else(|| lit(false));
+    let row_delete_expr = row_delete_pred.unwrap_or_else(|| lit(false));
     let insert_expr = insert_pred.unwrap_or_else(|| lit(false));
     let active_expr = target_present.or(insert_expr);
 
@@ -900,11 +1021,11 @@ fn build_default_merge_expansion(
     let projection_exprs = build_merge_projection(
         &options,
         target_schema,
-        source_schema,
         path_column,
         row_index_column,
+        row_delete_metadata_columns,
     )?;
-    trace!("projection exprs: {:?}", &projection_exprs);
+    trace!("projection exprs: {:?}", projection_exprs);
     let projected = LogicalPlanBuilder::from(filtered)
         .project(projection_exprs.clone())?
         .build()?;
@@ -913,11 +1034,14 @@ fn build_default_merge_expansion(
     // rewritten rows. Targeted rewrite can drop matched-but-unchanged rows from
     // untouched files, and conditional inserts can drop source-only rows. Aggregate
     // first so only one metric row flows to the writer.
+    // TODO: Let format adapters omit source metrics and cloned side plans they do not
+    // consume without adding format-specific policy to this shared expansion layer.
     let source_metric_projected = build_source_metric_plan(
         source_plan.clone(),
         target_schema,
         path_column,
         row_index_column,
+        row_delete_metadata_columns,
     )?;
     let projected = LogicalPlanBuilder::from(projected)
         .union(source_metric_projected)?
@@ -938,14 +1062,20 @@ fn build_default_merge_expansion(
         .project(vec![col(path_column).alias(path_column.to_string())])?
         .build()?;
 
-    let deletion_vector_plan = if let Some(row_index_column) = row_index_column {
+    let row_index_delete_plan = if let Some(row_index_column) = row_index_column {
+        let mut delete_projection = vec![
+            col(path_column).alias(path_column.to_string()),
+            col(row_index_column).alias(row_index_column.to_string()),
+        ];
+        delete_projection.extend(
+            row_delete_metadata_columns
+                .iter()
+                .map(|column| col(*column).alias((*column).to_string())),
+        );
         Some(
             LogicalPlanBuilder::from(join.as_ref().clone())
-                .filter(delete_expr)?
-                .project(vec![
-                    col(path_column).alias(path_column.to_string()),
-                    col(row_index_column).alias(row_index_column.to_string()),
-                ])?
+                .filter(row_delete_expr)?
+                .project(delete_projection)?
                 .build()?,
         )
     } else {
@@ -957,7 +1087,7 @@ fn build_default_merge_expansion(
     Ok(MergeExpansion {
         write_plan: projected.clone(),
         touched_files_plan: touched_plan,
-        deletion_vector_plan,
+        row_index_delete_plan,
         output_schema: command_schema,
         options,
     })
@@ -974,11 +1104,11 @@ fn append_presence_projection(
         .map(|f| Expr::Column(Column::from_name(f.name().clone())))
         .collect();
 
-    if let Some(path_name) = path_column {
-        if schema.index_of_column_by_name(None, path_name).is_none() {
-            let path_expr = lit(ScalarValue::Utf8(None));
-            exprs.push(path_expr.alias(path_name.to_string()));
-        }
+    if let Some(path_name) = path_column
+        && schema.index_of_column_by_name(None, path_name).is_none()
+    {
+        let path_expr = lit(ScalarValue::Utf8(None));
+        exprs.push(path_expr.alias(path_name.to_string()));
     }
 
     exprs.push(lit(true).alias(present_col));
@@ -1017,10 +1147,10 @@ fn can_fast_append_insert_only(
     };
 
     for clause in &options.not_matched_by_target_clauses {
-        if let Some(cond) = &clause.condition {
-            if references_target(&cond.expr)? {
-                return Ok(false);
-            }
+        if let Some(cond) = &clause.condition
+            && references_target(&cond.expr)?
+        {
+            return Ok(false);
         }
         match &clause.action {
             MergeNotMatchedByTargetAction::InsertAll => {}
@@ -1128,37 +1258,69 @@ fn row_level_data_operation_expr() -> Expr {
         .or(op.eq(lit(RowLevelOperationType::NotMatchedBySourceUpdate.as_i32())))
 }
 
+fn is_merge_metadata_column(
+    name: &str,
+    path_column: &str,
+    row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
+) -> bool {
+    name == path_column
+        || row_index_column.is_some_and(|column| name == column)
+        || row_delete_metadata_columns.contains(&name)
+        || name == TARGET_ROW_ID_COLUMN
+}
+
+fn merge_source_expr(options: &MergeIntoOptions, target_name: &str) -> Option {
+    options
+        .source_column_aliases
+        .iter()
+        .find(|(source_name, _)| {
+            merge_names_equal(source_name, target_name, options.case_sensitive)
+        })
+        .map(|(_, source_alias)| Expr::Column(Column::from_name(source_alias.clone())))
+}
+
+fn merge_default_expr(options: &MergeIntoOptions, target_name: &str) -> Option {
+    options
+        .default_column_exprs
+        .iter()
+        .find(|(column_name, _)| {
+            merge_names_equal(column_name, target_name, options.case_sensitive)
+        })
+        .map(|(_, expr)| expr.clone())
+}
+
+fn merge_insert_value(options: &MergeIntoOptions, target_name: &str) -> Expr {
+    merge_default_expr(options, target_name).unwrap_or_else(|| lit(ScalarValue::Null))
+}
+
+fn align_merge_value(
+    value: Expr,
+    target_field: &datafusion_common::arrow::datatypes::Field,
+) -> Expr {
+    // TODO: Validate casts against spark.sql.storeAssignmentPolicy.
+    cast(value, target_field.data_type().clone())
+}
+
 fn build_insert_only_projection(
     options: &MergeIntoOptions,
     target_schema: &DFSchemaRef,
-    source_schema: &DFSchemaRef,
     path_column: &str,
     row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
     operation_expr: Expr,
 ) -> Result> {
     // Match existing MERGE behavior: clause order determines first-match semantics.
     // Source rows that should not be written flow through as `Noop` rows for metrics.
     let mut projections = Vec::new();
 
-    // Source columns are prefixed with `__sail_src_`, so target field "id" maps
-    // to source column "__sail_src_id". Keys are lowercased for case-insensitive
-    // SQL resolution.
-    let source_exprs_by_name: HashMap = source_schema
-        .fields()
-        .iter()
-        .map(|f| {
-            (
-                f.name().to_ascii_lowercase(),
-                Expr::Column(Column::from_name(f.name().clone())),
-            )
-        })
-        .collect();
-
     for field in target_schema.fields().iter() {
-        if field.name() == path_column
-            || row_index_column.is_some_and(|c| field.name() == c)
-            || field.name() == TARGET_ROW_ID_COLUMN
-        {
+        if is_merge_metadata_column(
+            field.name(),
+            path_column,
+            row_index_column,
+            row_delete_metadata_columns,
+        ) {
             continue;
         }
         let name = field.name().clone();
@@ -1171,16 +1333,12 @@ fn build_insert_only_projection(
                 .map(|x| x.expr.clone())
                 .unwrap_or_else(|| lit(true));
             let value = match &clause.action {
-                MergeNotMatchedByTargetAction::InsertAll => source_exprs_by_name
-                    .get(&format!("__sail_src_{}", name.to_ascii_lowercase()))
-                    .cloned()
-                    .unwrap_or_else(|| lit(ScalarValue::Null)),
+                MergeNotMatchedByTargetAction::InsertAll => merge_source_expr(options, &name)
+                    .unwrap_or_else(|| merge_insert_value(options, &name)),
                 MergeNotMatchedByTargetAction::InsertColumns { columns, values } => {
-                    // If column not specified in this clause, it becomes NULL for this clause
-                    // (and must NOT fall through to later clauses).
-                    let mut out = lit(ScalarValue::Null);
+                    let mut out = merge_insert_value(options, &name);
                     for (col_name, expr) in columns.iter().zip(values.iter()) {
-                        if col_name.eq_ignore_ascii_case(&name) {
+                        if merge_names_equal(col_name, &name, options.case_sensitive) {
                             out = expr.clone();
                             break;
                         }
@@ -1188,7 +1346,7 @@ fn build_insert_only_projection(
                     out
                 }
             };
-            branches.push((pred, value));
+            branches.push((pred, align_merge_value(value, field)));
         }
 
         let when_then_expr = branches
@@ -1214,13 +1372,16 @@ fn build_insert_only_noop_projection(
     target_schema: &DFSchemaRef,
     path_column: &str,
     row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
 ) -> Result> {
     let mut projections = Vec::new();
     for field in target_schema.fields().iter() {
-        if field.name() == path_column
-            || row_index_column.is_some_and(|c| field.name() == c)
-            || field.name() == TARGET_ROW_ID_COLUMN
-        {
+        if is_merge_metadata_column(
+            field.name(),
+            path_column,
+            row_index_column,
+            row_delete_metadata_columns,
+        ) {
             continue;
         }
         let null_value = ScalarValue::try_new_null(field.data_type())?;
@@ -1235,6 +1396,7 @@ fn build_source_metric_plan(
     target_schema: &DFSchemaRef,
     path_column: &str,
     row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
 ) -> Result {
     let source_count = Expr::AggregateFunction(AggregateFunction {
         func: count_udaf(),
@@ -1260,9 +1422,12 @@ fn build_source_metric_plan(
             path_expr = Some(lit(null_value).alias(path_column.to_string()));
             continue;
         }
-        if row_index_column.is_some_and(|c| field.name() == c)
-            || field.name() == TARGET_ROW_ID_COLUMN
-        {
+        if is_merge_metadata_column(
+            field.name(),
+            path_column,
+            row_index_column,
+            row_delete_metadata_columns,
+        ) {
             continue;
         }
         let null_value = ScalarValue::try_new_null(field.data_type())?;
@@ -1273,6 +1438,24 @@ fn build_source_metric_plan(
         return plan_err!("MERGE source metric projection is missing required path column");
     };
     projections.push(path_expr);
+    if !row_delete_metadata_columns.is_empty() {
+        for metadata_column in row_index_column
+            .into_iter()
+            .chain(row_delete_metadata_columns.iter().copied())
+        {
+            let field = target_schema
+                .fields()
+                .iter()
+                .find(|field| field.name() == metadata_column)
+                .ok_or_else(|| {
+                    DataFusionError::Internal(format!(
+                        "MERGE source metric projection is missing metadata column '{metadata_column}'"
+                    ))
+                })?;
+            let null_value = ScalarValue::try_new_null(field.data_type())?;
+            projections.push(lit(null_value).alias(metadata_column.to_string()));
+        }
+    }
     projections.push(lit(RowLevelOperationType::SourceMetric.as_i32()).alias(OPERATION_COLUMN));
     projections.push(col(MERGE_SOURCE_METRIC_COLUMN).alias(MERGE_SOURCE_METRIC_COLUMN));
 
@@ -1448,20 +1631,83 @@ fn or_pred(existing: Option, expr: Expr) -> Option {
     })
 }
 
+fn first_matching_clause_predicates<'a>(
+    base_predicate: Expr,
+    conditions: impl IntoIterator>,
+) -> Vec {
+    let mut prior_match: Option = None;
+    conditions
+        .into_iter()
+        .map(|condition| {
+            let clause_match = base_predicate
+                .clone()
+                .and(
+                    condition
+                        .map(|condition| condition.expr.clone())
+                        .unwrap_or_else(|| lit(true)),
+                )
+                .is_true();
+            let selected = match &prior_match {
+                Some(prior_match) => clause_match.clone().and(not(prior_match.clone())),
+                None => clause_match.clone(),
+            };
+            prior_match = Some(match prior_match.take() {
+                Some(prior_match) => prior_match.or(clause_match),
+                None => clause_match,
+            });
+            selected
+        })
+        .collect()
+}
+
 fn build_merge_projection(
     options: &MergeIntoOptions,
     target_schema: &DFSchemaRef,
-    source_schema: &DFSchemaRef,
     path_column: &str,
     row_index_column: Option<&str>,
+    row_delete_metadata_columns: &[&str],
 ) -> Result> {
+    let matched_base = col(TARGET_PRESENT_COLUMN)
+        .is_not_null()
+        .and(col(SOURCE_PRESENT_COLUMN).is_not_null());
+    let matched_clause_predicates = first_matching_clause_predicates(
+        matched_base,
+        options
+            .matched_clauses
+            .iter()
+            .map(|clause| clause.condition.as_ref()),
+    );
+    let not_matched_by_source_base = col(TARGET_PRESENT_COLUMN)
+        .is_not_null()
+        .and(col(SOURCE_PRESENT_COLUMN).is_null());
+    let not_matched_by_source_clause_predicates = first_matching_clause_predicates(
+        not_matched_by_source_base,
+        options
+            .not_matched_by_source_clauses
+            .iter()
+            .map(|clause| clause.condition.as_ref()),
+    );
+    let not_matched_by_target_base = col(TARGET_PRESENT_COLUMN)
+        .is_null()
+        .and(col(SOURCE_PRESENT_COLUMN).is_not_null());
+    let not_matched_by_target_clause_predicates = first_matching_clause_predicates(
+        not_matched_by_target_base,
+        options
+            .not_matched_by_target_clauses
+            .iter()
+            .map(|clause| clause.condition.as_ref()),
+    );
+
     let mut cases: HashMap> = target_schema
         .fields()
         .iter()
-        .filter(|f| {
-            f.name() != path_column
-                && row_index_column.is_none_or(|c| f.name() != c)
-                && f.name() != TARGET_ROW_ID_COLUMN
+        .filter(|field| {
+            !is_merge_metadata_column(
+                field.name(),
+                path_column,
+                row_index_column,
+                row_delete_metadata_columns,
+            )
         })
         .map(|f| (f.name().clone(), Vec::new()))
         .collect();
@@ -1472,101 +1718,95 @@ fn build_merge_projection(
         .map(|f| (f.name().clone(), Expr::Column(Column::from_name(f.name()))))
         .collect();
 
-    let source_exprs_by_name: HashMap = source_schema
-        .fields()
+    for (clause, pred) in options
+        .matched_clauses
         .iter()
-        .map(|f| {
-            (
-                f.name().to_ascii_lowercase(),
-                Expr::Column(Column::from_name(f.name().clone())),
-            )
-        })
-        .collect();
-
-    // Find the source expression that corresponds to a target field by name.
-    // Source columns are prefixed with `__sail_src_`, so target field "id"
-    // maps to source column "__sail_src_id". Keys are lowercased for
-    // case-insensitive SQL resolution.
-    let source_expr_for_target = |target_name: &str| -> Expr {
-        let prefixed = format!("__sail_src_{}", target_name.to_ascii_lowercase());
-        source_exprs_by_name
-            .get(&prefixed)
-            .cloned()
-            .unwrap_or_else(|| lit(ScalarValue::Null))
-    };
-
-    for clause in &options.matched_clauses {
-        let mut pred = col(TARGET_PRESENT_COLUMN)
-            .is_not_null()
-            .and(col(SOURCE_PRESENT_COLUMN).is_not_null());
-        if let Some(cond) = &clause.condition {
-            pred = pred.and(cond.expr.clone());
-        }
+        .zip(&matched_clause_predicates)
+    {
         match &clause.action {
             MergeMatchedAction::Delete => {}
             MergeMatchedAction::UpdateAll => {
                 for field in target_schema.fields().iter() {
-                    let value = source_expr_for_target(field.name());
+                    let value = merge_source_expr(options, field.name())
+                        .or_else(|| target_exprs_by_name.get(field.name()).cloned())
+                        .unwrap_or_else(|| lit(ScalarValue::Null));
                     if let Some(v) = cases.get_mut(field.name()) {
-                        v.push((pred.clone(), value));
+                        v.push((pred.clone(), align_merge_value(value, field)));
                     }
                 }
             }
             MergeMatchedAction::UpdateSet(assignments) => {
                 for assignment in assignments {
-                    let resolved =
-                        resolve_target_column(assignment.column.as_str(), target_schema)?;
+                    let resolved = resolve_target_column(
+                        assignment.column.as_str(),
+                        target_schema,
+                        options.case_sensitive,
+                    )?;
                     if let Some(v) = cases.get_mut(&resolved) {
-                        v.push((pred.clone(), assignment.value.clone()));
+                        let field = merge_target_field(target_schema, &resolved)?;
+                        v.push((
+                            pred.clone(),
+                            align_merge_value(assignment.value.clone(), field),
+                        ));
                     }
                 }
             }
         }
     }
 
-    for clause in &options.not_matched_by_source_clauses {
-        let mut pred = col(TARGET_PRESENT_COLUMN)
-            .is_not_null()
-            .and(col(SOURCE_PRESENT_COLUMN).is_null());
-        if let Some(cond) = &clause.condition {
-            pred = pred.and(cond.expr.clone());
-        }
+    for (clause, pred) in options
+        .not_matched_by_source_clauses
+        .iter()
+        .zip(¬_matched_by_source_clause_predicates)
+    {
         match &clause.action {
             MergeNotMatchedBySourceAction::Delete => {}
             MergeNotMatchedBySourceAction::UpdateSet(assignments) => {
                 for assignment in assignments {
-                    let resolved =
-                        resolve_target_column(assignment.column.as_str(), target_schema)?;
+                    let resolved = resolve_target_column(
+                        assignment.column.as_str(),
+                        target_schema,
+                        options.case_sensitive,
+                    )?;
                     if let Some(v) = cases.get_mut(&resolved) {
-                        v.push((pred.clone(), assignment.value.clone()));
+                        let field = merge_target_field(target_schema, &resolved)?;
+                        v.push((
+                            pred.clone(),
+                            align_merge_value(assignment.value.clone(), field),
+                        ));
                     }
                 }
             }
         }
     }
 
-    for clause in &options.not_matched_by_target_clauses {
-        let mut pred = col(TARGET_PRESENT_COLUMN)
-            .is_null()
-            .and(col(SOURCE_PRESENT_COLUMN).is_not_null());
-        if let Some(cond) = &clause.condition {
-            pred = pred.and(cond.expr.clone());
-        }
-
+    for (clause, pred) in options
+        .not_matched_by_target_clauses
+        .iter()
+        .zip(¬_matched_by_target_clause_predicates)
+    {
         match &clause.action {
             MergeNotMatchedByTargetAction::InsertAll => {
                 for field in target_schema.fields().iter() {
-                    let value = source_expr_for_target(field.name());
+                    let value = merge_source_expr(options, field.name())
+                        .unwrap_or_else(|| merge_insert_value(options, field.name()));
                     if let Some(v) = cases.get_mut(field.name()) {
-                        v.push((pred.clone(), value));
+                        v.push((pred.clone(), align_merge_value(value, field)));
                     }
                 }
             }
             MergeNotMatchedByTargetAction::InsertColumns { columns, values } => {
-                for (col_name, value) in columns.iter().zip(values.iter()) {
-                    let resolved = resolve_target_column(col_name, target_schema)?;
-                    if let Some(v) = cases.get_mut(&resolved) {
-                        v.push((pred.clone(), value.clone()));
+                for field in target_schema.fields().iter() {
+                    if let Some(v) = cases.get_mut(field.name()) {
+                        let value = columns
+                            .iter()
+                            .zip(values.iter())
+                            .find(|(column_name, _)| {
+                                merge_names_equal(column_name, field.name(), options.case_sensitive)
+                            })
+                            .map(|(_, value)| value.clone())
+                            .unwrap_or_else(|| merge_insert_value(options, field.name()));
+                        v.push((pred.clone(), align_merge_value(value, field)));
                     }
                 }
             }
@@ -1575,10 +1815,12 @@ fn build_merge_projection(
 
     let mut projections = Vec::new();
     for field in target_schema.fields() {
-        if field.name() == path_column
-            || row_index_column.is_some_and(|c| field.name() == c)
-            || field.name() == TARGET_ROW_ID_COLUMN
-        {
+        if is_merge_metadata_column(
+            field.name(),
+            path_column,
+            row_index_column,
+            row_delete_metadata_columns,
+        ) {
             continue;
         }
         let name = field.name();
@@ -1608,45 +1850,17 @@ fn build_merge_projection(
     // Preserve the file path column so downstream physical planning can implement
     // targeted rewrite (filter writer input to touched files, while keeping inserts).
     projections.push(col(path_column).alias(path_column.to_string()));
-
-    let mut matched_update_pred: Option = None;
-    let mut not_matched_by_source_update_pred: Option = None;
-    let mut matched_delete_pred: Option = None;
-    let mut not_matched_by_source_delete_pred: Option = None;
-
-    for clause in &options.matched_clauses {
-        let mut pred = col(TARGET_PRESENT_COLUMN)
-            .is_not_null()
-            .and(col(SOURCE_PRESENT_COLUMN).is_not_null());
-        if let Some(cond) = &clause.condition {
-            pred = pred.and(cond.expr.clone());
-        }
-        match &clause.action {
-            MergeMatchedAction::UpdateAll | MergeMatchedAction::UpdateSet(_) => {
-                matched_update_pred = or_pred(matched_update_pred, pred);
-            }
-            MergeMatchedAction::Delete => {
-                matched_delete_pred = or_pred(matched_delete_pred, pred);
-            }
-        }
-    }
-    for clause in &options.not_matched_by_source_clauses {
-        let mut pred = col(TARGET_PRESENT_COLUMN)
-            .is_not_null()
-            .and(col(SOURCE_PRESENT_COLUMN).is_null());
-        if let Some(cond) = &clause.condition {
-            pred = pred.and(cond.expr.clone());
-        }
-        match &clause.action {
-            MergeNotMatchedBySourceAction::UpdateSet(_) => {
-                not_matched_by_source_update_pred =
-                    or_pred(not_matched_by_source_update_pred, pred);
-            }
-            MergeNotMatchedBySourceAction::Delete => {
-                not_matched_by_source_delete_pred =
-                    or_pred(not_matched_by_source_delete_pred, pred);
-            }
+    // A format that supplies delete metadata consumes row deletes from this projection,
+    // so keep the row identity next to that metadata for the downstream writer.
+    if !row_delete_metadata_columns.is_empty() {
+        if let Some(row_index_column) = row_index_column {
+            projections.push(col(row_index_column).alias(row_index_column.to_string()));
         }
+        projections.extend(
+            row_delete_metadata_columns
+                .iter()
+                .map(|column| col(*column).alias((*column).to_string())),
+        );
     }
 
     // Append the operation type column so downstream writers know per-row intent.
@@ -1657,30 +1871,45 @@ fn build_merge_projection(
     // each writer's local plan shape.
     let insert_op = lit(RowLevelOperationType::Insert.as_i32());
     let copy_op = lit(RowLevelOperationType::Copy.as_i32());
+    let mut operation_branches = vec![(
+        Box::new(col(TARGET_PRESENT_COLUMN).is_null()),
+        Box::new(insert_op),
+    )];
+    operation_branches.extend(
+        options
+            .matched_clauses
+            .iter()
+            .zip(matched_clause_predicates)
+            .map(|(clause, predicate)| {
+                let operation = match &clause.action {
+                    MergeMatchedAction::UpdateAll | MergeMatchedAction::UpdateSet(_) => {
+                        RowLevelOperationType::MatchedUpdate
+                    }
+                    MergeMatchedAction::Delete => RowLevelOperationType::MatchedDelete,
+                };
+                (Box::new(predicate), Box::new(lit(operation.as_i32())))
+            }),
+    );
+    operation_branches.extend(
+        options
+            .not_matched_by_source_clauses
+            .iter()
+            .zip(not_matched_by_source_clause_predicates)
+            .map(|(clause, predicate)| {
+                let operation = match &clause.action {
+                    MergeNotMatchedBySourceAction::UpdateSet(_) => {
+                        RowLevelOperationType::NotMatchedBySourceUpdate
+                    }
+                    MergeNotMatchedBySourceAction::Delete => {
+                        RowLevelOperationType::NotMatchedBySourceDelete
+                    }
+                };
+                (Box::new(predicate), Box::new(lit(operation.as_i32())))
+            }),
+    );
     let op_expr = Expr::Case(Case {
         expr: None,
-        when_then_expr: vec![
-            (
-                Box::new(col(TARGET_PRESENT_COLUMN).is_null()),
-                Box::new(insert_op),
-            ),
-            (
-                Box::new(matched_update_pred.unwrap_or_else(|| lit(false))),
-                Box::new(lit(RowLevelOperationType::MatchedUpdate.as_i32())),
-            ),
-            (
-                Box::new(not_matched_by_source_update_pred.unwrap_or_else(|| lit(false))),
-                Box::new(lit(RowLevelOperationType::NotMatchedBySourceUpdate.as_i32())),
-            ),
-            (
-                Box::new(matched_delete_pred.unwrap_or_else(|| lit(false))),
-                Box::new(lit(RowLevelOperationType::MatchedDelete.as_i32())),
-            ),
-            (
-                Box::new(not_matched_by_source_delete_pred.unwrap_or_else(|| lit(false))),
-                Box::new(lit(RowLevelOperationType::NotMatchedBySourceDelete.as_i32())),
-            ),
-        ],
+        when_then_expr: operation_branches,
         else_expr: Some(Box::new(copy_op)),
     });
     projections.push(op_expr.alias(OPERATION_COLUMN));
@@ -1723,11 +1952,31 @@ fn combine_rewrite_preds(matched: Vec, not_matched_by_source: Vec) -
     combine_disjunction(&preds)
 }
 
-fn resolve_target_column(column: &str, target_schema: &DFSchemaRef) -> Result {
+fn merge_target_field<'a>(
+    target_schema: &'a DFSchemaRef,
+    column: &str,
+) -> Result<&'a datafusion_common::arrow::datatypes::Field> {
+    target_schema
+        .fields()
+        .iter()
+        .find(|field| field.name() == column)
+        .map(AsRef::as_ref)
+        .ok_or_else(|| {
+            DataFusionError::Plan(format!(
+                "unable to resolve column {column} in MERGE target projection"
+            ))
+        })
+}
+
+fn resolve_target_column(
+    column: &str,
+    target_schema: &DFSchemaRef,
+    case_sensitive: bool,
+) -> Result {
     let matches = target_schema
         .fields()
         .iter()
-        .filter(|f| f.name().eq_ignore_ascii_case(column))
+        .filter(|field| merge_names_equal(field.name(), column, case_sensitive))
         .collect::>();
     if matches.len() != 1 {
         return plan_err!("unable to resolve column {column} in MERGE target projection");
@@ -1741,17 +1990,16 @@ fn rewrite_merge_columns(
     source_map: &HashMap,
 ) -> Result {
     expr.transform(|expr| {
-        if let Expr::Column(col) = &expr {
-            if let Some(new_name) = target_map
+        if let Expr::Column(col) = &expr
+            && let Some(new_name) = target_map
                 .get(&col.name)
                 .or_else(|| source_map.get(&col.name))
-            {
-                return Ok(Transformed::yes(Expr::Column(Column {
-                    relation: None,
-                    name: new_name.clone(),
-                    spans: col.spans.clone(),
-                })));
-            }
+        {
+            return Ok(Transformed::yes(Expr::Column(Column {
+                relation: None,
+                name: new_name.clone(),
+                spans: col.spans.clone(),
+            })));
         }
         Ok(Transformed::no(expr))
     })
diff --git a/crates/sail-logical-plan/src/remote_checkpoint.rs b/crates/sail-logical-plan/src/remote_checkpoint.rs
new file mode 100644
index 000000000..93274a2f7
--- /dev/null
+++ b/crates/sail-logical-plan/src/remote_checkpoint.rs
@@ -0,0 +1,134 @@
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion_common::{DFSchema, DFSchemaRef, Result, internal_datafusion_err};
+use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
+use educe::Educe;
+
+#[derive(Clone, Debug, Eq, PartialEq, Hash, Educe)]
+#[educe(PartialOrd)]
+pub struct RemoteCheckpointRelationNode {
+    relation_id: String,
+    #[educe(PartialOrd(ignore))]
+    schema: DFSchemaRef,
+}
+
+impl RemoteCheckpointRelationNode {
+    pub fn try_new(relation_id: String, schema: SchemaRef) -> Result {
+        Ok(Self {
+            relation_id,
+            schema: Arc::new(DFSchema::try_from(schema.as_ref().clone())?),
+        })
+    }
+
+    pub fn relation_id(&self) -> &str {
+        &self.relation_id
+    }
+}
+
+impl UserDefinedLogicalNodeCore for RemoteCheckpointRelationNode {
+    fn name(&self) -> &str {
+        "RemoteCheckpointRelation"
+    }
+
+    fn inputs(&self) -> Vec<&LogicalPlan> {
+        vec![]
+    }
+
+    fn schema(&self) -> &DFSchemaRef {
+        &self.schema
+    }
+
+    fn expressions(&self) -> Vec {
+        vec![]
+    }
+
+    fn fmt_for_explain(&self, f: &mut Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "RemoteCheckpointRelation: relation_id={}",
+            self.relation_id
+        )
+    }
+
+    fn with_exprs_and_inputs(&self, exprs: Vec, inputs: Vec) -> Result {
+        if !exprs.is_empty() || !inputs.is_empty() {
+            return Err(internal_datafusion_err!(
+                "RemoteCheckpointRelation does not accept expressions or inputs"
+            ));
+        }
+        Ok(self.clone())
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Hash, Educe)]
+#[educe(PartialOrd)]
+pub struct RemoteCheckpointCommandNode {
+    relation_id: String,
+    input: Arc,
+    #[educe(PartialOrd(ignore))]
+    logical_schema: SchemaRef,
+}
+
+impl RemoteCheckpointCommandNode {
+    pub fn new(relation_id: String, input: Arc, logical_schema: SchemaRef) -> Self {
+        Self {
+            relation_id,
+            input,
+            logical_schema,
+        }
+    }
+
+    pub fn relation_id(&self) -> &str {
+        &self.relation_id
+    }
+
+    pub fn logical_schema(&self) -> &SchemaRef {
+        &self.logical_schema
+    }
+}
+
+impl UserDefinedLogicalNodeCore for RemoteCheckpointCommandNode {
+    fn name(&self) -> &str {
+        "RemoteCheckpointCommand"
+    }
+
+    fn inputs(&self) -> Vec<&LogicalPlan> {
+        vec![self.input.as_ref()]
+    }
+
+    fn schema(&self) -> &DFSchemaRef {
+        DFSchema::empty_ref()
+    }
+
+    fn expressions(&self) -> Vec {
+        vec![]
+    }
+
+    fn fmt_for_explain(&self, f: &mut Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "RemoteCheckpointCommand: relation_id={}",
+            self.relation_id
+        )
+    }
+
+    fn with_exprs_and_inputs(&self, exprs: Vec, inputs: Vec) -> Result {
+        if !exprs.is_empty() || inputs.len() != 1 {
+            return Err(internal_datafusion_err!(
+                "RemoteCheckpointCommand requires one input and no expressions"
+            ));
+        }
+        Ok(Self {
+            input: Arc::new(inputs.into_iter().next().ok_or_else(|| {
+                internal_datafusion_err!("RemoteCheckpointCommand input is missing")
+            })?),
+            ..self.clone()
+        })
+    }
+
+    fn necessary_children_exprs(&self, _output_columns: &[usize]) -> Option>> {
+        Some(vec![(0..self.input.schema().fields().len()).collect()])
+    }
+}
diff --git a/crates/sail-logical-plan/src/repartition.rs b/crates/sail-logical-plan/src/repartition.rs
index 84c09953c..51787524a 100644
--- a/crates/sail-logical-plan/src/repartition.rs
+++ b/crates/sail-logical-plan/src/repartition.rs
@@ -2,8 +2,8 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use datafusion::logical_expr::LogicalPlan;
-use datafusion_common::{plan_err, DFSchemaRef, Result};
-use datafusion_expr::{expr_vec_fmt, Expr, UserDefinedLogicalNodeCore};
+use datafusion_common::{DFSchemaRef, Result, plan_err};
+use datafusion_expr::{Expr, UserDefinedLogicalNodeCore, expr_vec_fmt};
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)]
 pub enum ExplicitRepartitionKind {
diff --git a/crates/sail-logical-plan/src/schema_pivot.rs b/crates/sail-logical-plan/src/schema_pivot.rs
index 5c9d6237c..f91a27574 100644
--- a/crates/sail-logical-plan/src/schema_pivot.rs
+++ b/crates/sail-logical-plan/src/schema_pivot.rs
@@ -3,7 +3,7 @@ use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
 use datafusion::arrow::datatypes::{DataType, Field};
-use datafusion_common::{plan_err, DFSchema, DFSchemaRef, Result};
+use datafusion_common::{DFSchema, DFSchemaRef, Result, plan_err};
 use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
 use educe::Educe;
 use sail_common_datafusion::utils::items::ItemTaker;
diff --git a/crates/sail-logical-plan/src/show_string.rs b/crates/sail-logical-plan/src/show_string.rs
index 27cc8693d..1ef2e1de6 100644
--- a/crates/sail-logical-plan/src/show_string.rs
+++ b/crates/sail-logical-plan/src/show_string.rs
@@ -9,7 +9,7 @@ use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion_common::{DFSchema, DFSchemaRef, Result};
 use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
 use educe::Educe;
-use sail_common::string::escape_meta_characters;
+use sail_common::utils::string::escape_meta_characters;
 use sail_common_datafusion::display::{ArrayFormatter, FormatOptions};
 use sail_common_datafusion::utils::items::ItemTaker;
 
diff --git a/crates/sail-logical-plan/src/sort.rs b/crates/sail-logical-plan/src/sort.rs
index cf937076b..1c2449ec2 100644
--- a/crates/sail-logical-plan/src/sort.rs
+++ b/crates/sail-logical-plan/src/sort.rs
@@ -6,6 +6,106 @@ use datafusion_expr::expr::{Expr, Sort};
 use datafusion_expr::{LogicalPlan, UserDefinedLogicalNodeCore};
 use sail_common_datafusion::utils::items::ItemTaker;
 
+#[derive(Clone, Debug, Eq, PartialEq, Hash, PartialOrd)]
+pub struct RequiredSortNode {
+    input: Arc,
+    sort_expr: Vec,
+    fetch: Option,
+    preserve_partitioning: bool,
+}
+
+impl RequiredSortNode {
+    pub fn new(
+        input: Arc,
+        sort_expr: Vec,
+        fetch: Option,
+        preserve_partitioning: bool,
+    ) -> Self {
+        Self {
+            input,
+            sort_expr,
+            fetch,
+            preserve_partitioning,
+        }
+    }
+
+    pub fn fetch(&self) -> Option {
+        self.fetch
+    }
+
+    pub fn preserve_partitioning(&self) -> bool {
+        self.preserve_partitioning
+    }
+
+    pub fn sort_expr(&self) -> &[Sort] {
+        &self.sort_expr
+    }
+}
+
+impl UserDefinedLogicalNodeCore for RequiredSortNode {
+    fn name(&self) -> &str {
+        "RequiredSort"
+    }
+
+    fn inputs(&self) -> Vec<&LogicalPlan> {
+        vec![self.input.as_ref()]
+    }
+
+    fn schema(&self) -> &DFSchemaRef {
+        self.input.schema()
+    }
+
+    fn expressions(&self) -> Vec {
+        self.sort_expr.iter().map(|s| s.expr.clone()).collect()
+    }
+
+    fn fmt_for_explain(&self, f: &mut Formatter) -> std::fmt::Result {
+        write!(f, "RequiredSort: ")?;
+        for (i, e) in self.sort_expr.iter().enumerate() {
+            if i > 0 {
+                write!(f, ", ")?;
+            }
+            write!(f, "{e}")?;
+        }
+        if let Some(a) = self.fetch {
+            write!(f, ", fetch={a}")?;
+        }
+        Ok(())
+    }
+
+    fn with_exprs_and_inputs(&self, exprs: Vec, inputs: Vec) -> Result {
+        if exprs.len() != self.sort_expr.len() {
+            return Err(datafusion_common::DataFusionError::Internal(format!(
+                "RequiredSortNode: expected {} expressions, got {}",
+                self.sort_expr.len(),
+                exprs.len()
+            )));
+        }
+
+        let sort_expr = self
+            .sort_expr
+            .iter()
+            .zip(exprs)
+            .map(|(old_sort, new_expr)| Sort {
+                expr: new_expr,
+                asc: old_sort.asc,
+                nulls_first: old_sort.nulls_first,
+            })
+            .collect();
+
+        Ok(Self {
+            input: Arc::new(inputs.one()?),
+            sort_expr,
+            fetch: self.fetch,
+            preserve_partitioning: self.preserve_partitioning,
+        })
+    }
+
+    fn necessary_children_exprs(&self, output_columns: &[usize]) -> Option>> {
+        Some(vec![output_columns.to_vec()])
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq, Hash, PartialOrd)]
 pub struct SortWithinPartitionsNode {
     input: Arc,
@@ -26,6 +126,10 @@ impl SortWithinPartitionsNode {
         self.fetch
     }
 
+    pub fn input(&self) -> &Arc {
+        &self.input
+    }
+
     pub fn sort_expr(&self) -> &[Sort] {
         &self.sort_expr
     }
diff --git a/crates/sail-logical-plan/src/streaming/collector.rs b/crates/sail-logical-plan/src/streaming/collector.rs
index 80585b402..bf4c1f610 100644
--- a/crates/sail-logical-plan/src/streaming/collector.rs
+++ b/crates/sail-logical-plan/src/streaming/collector.rs
@@ -2,7 +2,7 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use datafusion::logical_expr::LogicalPlan;
-use datafusion_common::{plan_err, DFSchema, DFSchemaRef, Result};
+use datafusion_common::{DFSchema, DFSchemaRef, Result, plan_err};
 use datafusion_expr::{Expr, UserDefinedLogicalNodeCore};
 use educe::Educe;
 use sail_common_datafusion::streaming::event::schema::try_from_flow_event_schema;
diff --git a/crates/sail-logical-plan/src/streaming/limit.rs b/crates/sail-logical-plan/src/streaming/limit.rs
index 7eb293f33..636dad38f 100644
--- a/crates/sail-logical-plan/src/streaming/limit.rs
+++ b/crates/sail-logical-plan/src/streaming/limit.rs
@@ -2,7 +2,7 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use datafusion::logical_expr::LogicalPlan;
-use datafusion_common::{plan_err, DFSchemaRef, Result};
+use datafusion_common::{DFSchemaRef, Result, plan_err};
 use datafusion_expr::{Expr, UserDefinedLogicalNodeCore};
 
 /// A logical plan node that limits the number of retractable rows in a stream
diff --git a/crates/sail-logical-plan/src/streaming/source_adapter.rs b/crates/sail-logical-plan/src/streaming/source_adapter.rs
index a42a0042d..8cdeb5042 100644
--- a/crates/sail-logical-plan/src/streaming/source_adapter.rs
+++ b/crates/sail-logical-plan/src/streaming/source_adapter.rs
@@ -2,7 +2,7 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use datafusion::logical_expr::LogicalPlan;
-use datafusion_common::{plan_err, DFSchema, DFSchemaRef, Result};
+use datafusion_common::{DFSchema, DFSchemaRef, Result, plan_err};
 use datafusion_expr::{Expr, UserDefinedLogicalNodeCore};
 use educe::Educe;
 use sail_common_datafusion::streaming::event::schema::to_flow_event_schema;
diff --git a/crates/sail-logical-plan/src/streaming/source_wrapper.rs b/crates/sail-logical-plan/src/streaming/source_wrapper.rs
index 6616cfb68..102473965 100644
--- a/crates/sail-logical-plan/src/streaming/source_wrapper.rs
+++ b/crates/sail-logical-plan/src/streaming/source_wrapper.rs
@@ -3,7 +3,7 @@ use std::hash::Hash;
 use std::sync::Arc;
 
 use datafusion::logical_expr::LogicalPlan;
-use datafusion_common::{plan_err, DFSchema, DFSchemaRef, Result, TableReference};
+use datafusion_common::{DFSchema, DFSchemaRef, Result, TableReference, plan_err};
 use datafusion_expr::{Expr, UserDefinedLogicalNodeCore};
 use educe::Educe;
 use sail_common_datafusion::rename::expression::expression_before_rename;
diff --git a/crates/sail-object-store/Cargo.toml b/crates/sail-object-store/Cargo.toml
index f6851c8a3..a4bb48d91 100644
--- a/crates/sail-object-store/Cargo.toml
+++ b/crates/sail-object-store/Cargo.toml
@@ -8,7 +8,6 @@ workspace = true
 
 [dependencies]
 sail-common = { path = "../sail-common" }
-sail-telemetry = { path = "../sail-telemetry" }
 
 object_store = { workspace = true }
 aws-config = { workspace = true }
diff --git a/crates/sail-object-store/src/hugging_face.rs b/crates/sail-object-store/src/hugging_face.rs
index dac4423b0..50a37fe58 100644
--- a/crates/sail-object-store/src/hugging_face.rs
+++ b/crates/sail-object-store/src/hugging_face.rs
@@ -12,8 +12,8 @@ use log::debug;
 use object_store::local::LocalFileSystem;
 use object_store::path::Path;
 use object_store::{
-    path, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta,
-    ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult,
+    GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, path,
 };
 use regex::Regex;
 use reqwest::StatusCode;
diff --git a/crates/sail-object-store/src/layers/lazy.rs b/crates/sail-object-store/src/layers/lazy.rs
index 0f06fbfc6..d3c8b3a53 100644
--- a/crates/sail-object-store/src/layers/lazy.rs
+++ b/crates/sail-object-store/src/layers/lazy.rs
@@ -4,8 +4,8 @@ use std::future::Future;
 use std::ops::Range;
 use std::sync::Arc;
 
-use futures::stream::BoxStream;
 use futures::StreamExt;
+use futures::stream::BoxStream;
 use object_store::path::Path;
 use object_store::{
     CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
diff --git a/crates/sail-object-store/src/layers/logging.rs b/crates/sail-object-store/src/layers/logging.rs
index 8a0d6a603..f4b329fb7 100644
--- a/crates/sail-object-store/src/layers/logging.rs
+++ b/crates/sail-object-store/src/layers/logging.rs
@@ -4,17 +4,15 @@ use std::sync::Arc;
 
 use fastrace::Span;
 use fastrace_futures::StreamExt;
-use futures::stream::BoxStream;
 use futures::FutureExt;
+use futures::stream::BoxStream;
 use log::debug;
 use object_store::path::Path;
 use object_store::{
     CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
     PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result, UploadPart,
 };
-use sail_telemetry::common::SpanAttribute;
-use sail_telemetry::futures::TracingFutureExt;
-use sail_telemetry::recorder::record_error;
+use sail_common::telemetry::{SpanAttribute, TracingFutureExt, record_error};
 use tonic::codegen::Bytes;
 
 #[derive(Debug)]
diff --git a/crates/sail-object-store/src/layers/runtime.rs b/crates/sail-object-store/src/layers/runtime.rs
index 86acbd223..a2318c381 100644
--- a/crates/sail-object-store/src/layers/runtime.rs
+++ b/crates/sail-object-store/src/layers/runtime.rs
@@ -13,7 +13,7 @@ use object_store::{
     UploadPart,
 };
 use tokio::runtime::Handle;
-use tokio::sync::{mpsc, Mutex};
+use tokio::sync::{Mutex, mpsc};
 use tokio_stream::wrappers::ReceiverStream;
 use tonic::codegen::Bytes;
 
diff --git a/crates/sail-object-store/src/lib.rs b/crates/sail-object-store/src/lib.rs
index c9bc0541a..deb0b1647 100644
--- a/crates/sail-object-store/src/lib.rs
+++ b/crates/sail-object-store/src/lib.rs
@@ -1,6 +1,10 @@
 mod hugging_face;
 mod layers;
+mod path;
 mod registry;
 mod s3;
 
+pub use path::{
+    ResolvedObjectStorePath, delete_object_store_prefix_objects, resolve_object_store_path,
+};
 pub use registry::DynamicObjectStoreRegistry;
diff --git a/crates/sail-object-store/src/path.rs b/crates/sail-object-store/src/path.rs
new file mode 100644
index 000000000..ca4d01a9c
--- /dev/null
+++ b/crates/sail-object-store/src/path.rs
@@ -0,0 +1,114 @@
+use std::sync::Arc;
+
+use datafusion::datasource::listing::ListingTableUrl;
+use datafusion::execution::object_store::ObjectStoreUrl;
+use datafusion::execution::runtime_env::RuntimeEnv;
+use datafusion_common::{DataFusionError, Result};
+use futures::StreamExt;
+use object_store::ObjectStore;
+use object_store::path::Path;
+
+#[derive(Clone)]
+pub struct ResolvedObjectStorePath {
+    object_store_url: ObjectStoreUrl,
+    prefix: Path,
+    store: Arc,
+}
+
+impl ResolvedObjectStorePath {
+    pub fn object_store_url(&self) -> &ObjectStoreUrl {
+        &self.object_store_url
+    }
+
+    pub fn prefix(&self) -> &Path {
+        &self.prefix
+    }
+
+    pub fn store(&self) -> &Arc {
+        &self.store
+    }
+}
+
+pub async fn delete_object_store_prefix_objects(
+    store: &dyn ObjectStore,
+    prefix: &Path,
+) -> Result<()> {
+    let locations = store
+        .list(Some(prefix))
+        .map(|result| result.map(|object| object.location));
+    let mut deleted = store.delete_stream(Box::pin(locations));
+    while let Some(result) = deleted.next().await {
+        match result {
+            Ok(_) | Err(object_store::Error::NotFound { .. }) => {}
+            Err(error) => return Err(DataFusionError::ObjectStore(Box::new(error))),
+        }
+    }
+    Ok(())
+}
+
+pub fn resolve_object_store_path(
+    runtime_env: &RuntimeEnv,
+    path: &str,
+) -> Result {
+    let directory = format!("{}/", path.trim_end_matches('/'));
+    let url = ListingTableUrl::parse(&directory)?;
+    let object_store_url = url.object_store();
+    let store = runtime_env.object_store(&object_store_url)?;
+    Ok(ResolvedObjectStorePath {
+        object_store_url,
+        prefix: url.prefix().clone(),
+        store,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use datafusion::execution::runtime_env::RuntimeEnv;
+    use object_store::memory::InMemory;
+    use object_store::{ObjectStoreExt, PutPayload};
+    use url::Url;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn resolved_path_puts_and_deletes_only_its_prefix() -> Result<()> {
+        let runtime_env = RuntimeEnv::default();
+        runtime_env.register_object_store(
+            &Url::parse("memory:///")
+                .map_err(|error| DataFusionError::External(Box::new(error)))?,
+            Arc::new(InMemory::new()),
+        );
+        let resolved = resolve_object_store_path(&runtime_env, "memory:///checkpoint")?;
+        let first = resolved.prefix().clone().join("part-00000.parquet");
+        let nested = resolved
+            .prefix()
+            .clone()
+            .join("nested")
+            .join("part-00001.parquet");
+        let outside = Path::from("outside.parquet");
+
+        for (location, bytes) in [
+            (&first, b"first".as_slice()),
+            (&nested, b"nested".as_slice()),
+            (&outside, b"outside".as_slice()),
+        ] {
+            resolved
+                .store()
+                .put(location, PutPayload::from(bytes.to_vec()))
+                .await
+                .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?;
+        }
+        delete_object_store_prefix_objects(resolved.store().as_ref(), resolved.prefix()).await?;
+
+        assert!(matches!(
+            resolved.store().head(&first).await,
+            Err(object_store::Error::NotFound { .. })
+        ));
+        assert!(matches!(
+            resolved.store().head(&nested).await,
+            Err(object_store::Error::NotFound { .. })
+        ));
+        assert!(resolved.store().head(&outside).await.is_ok());
+        Ok(())
+    }
+}
diff --git a/crates/sail-object-store/src/registry.rs b/crates/sail-object-store/src/registry.rs
index 248b952f3..1fec6a974 100644
--- a/crates/sail-object-store/src/registry.rs
+++ b/crates/sail-object-store/src/registry.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use dashmap::DashMap;
 use datafusion::execution::object_store::ObjectStoreRegistry;
-use datafusion_common::{plan_datafusion_err, Result};
+use datafusion_common::{Result, plan_datafusion_err};
 #[cfg(feature = "hdfs")]
 use hdfs_native_object_store::HdfsObjectStoreBuilder;
 use log::debug;
@@ -135,6 +135,22 @@ fn get_dynamic_object_store(url: &Url) -> object_store::Result {
+            let url = url.clone();
+            let store = LazyObjectStore::new(move || {
+                let url = url.clone();
+                async move { get_s3_object_store(&url).await }
+            });
+            Arc::new(store)
+        }
+        _ if is_aliyun_oss_url(url) => {
+            let url = url.clone();
+            let store = LazyObjectStore::new(move || {
+                let url = url.clone();
+                async move { get_s3_object_store(&url).await }
+            });
+            Arc::new(store)
+        }
         _ => {
             let (scheme, _path) = ObjectStoreScheme::parse(url)?;
             let store: Arc = match scheme {
@@ -178,7 +194,7 @@ fn get_dynamic_object_store(url: &Url) -> object_store::Result object_store::Result bool {
+    if !matches!(url.scheme(), "http" | "https") {
+        return false;
+    }
+    let Some(host) = url.host_str() else {
+        return false;
+    };
+    let labels = host.split('.').collect::>();
+    matches!(
+        labels.as_slice(),
+        [_, "s3", region, "aliyuncs", "com"] | ["s3", region, "aliyuncs", "com"]
+            if region.starts_with("oss-")
+    )
+}
+
 // The following implementations are basic for now just to get preliminary functionality.
 pub async fn get_azure_object_store(url: &Url) -> object_store::Result {
     MicrosoftAzureBuilder::from_env()
@@ -218,13 +249,13 @@ mod tests {
     use std::sync::Arc;
 
     use datafusion::execution::object_store::ObjectStoreRegistry;
-    use object_store::memory::InMemory;
     use object_store::ObjectStore;
+    use object_store::memory::InMemory;
     use sail_common::runtime::RuntimeHandle;
     use tokio::runtime::Handle;
     use url::Url;
 
-    use super::{DynamicObjectStoreRegistry, ObjectStoreKey};
+    use super::{DynamicObjectStoreRegistry, ObjectStoreKey, is_aliyun_oss_url};
 
     #[test]
     fn object_store_key_separates_session_fingerprints() {
@@ -266,4 +297,17 @@ mod tests {
         let default_store = registry.get_store(&url).unwrap();
         assert!(Arc::ptr_eq(&fallback, &default_store));
     }
+
+    #[test]
+    fn aliyun_oss_url_detection_matches_s3_compatible_endpoints() {
+        assert!(is_aliyun_oss_url(
+            &Url::parse("https://bucket.s3.oss-cn-hangzhou.aliyuncs.com/path").unwrap()
+        ));
+        assert!(is_aliyun_oss_url(
+            &Url::parse("https://s3.oss-cn-hangzhou.aliyuncs.com/bucket/path").unwrap()
+        ));
+        assert!(!is_aliyun_oss_url(
+            &Url::parse("https://bucket.oss-cn-hangzhou.aliyuncs.com/path").unwrap()
+        ));
+    }
 }
diff --git a/crates/sail-object-store/src/s3.rs b/crates/sail-object-store/src/s3.rs
index 3e23dc2fd..ad0037e31 100644
--- a/crates/sail-object-store/src/s3.rs
+++ b/crates/sail-object-store/src/s3.rs
@@ -4,8 +4,8 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use aws_config::identity::IdentityCache;
 use aws_config::{BehaviorVersion, SdkConfig};
-use aws_credential_types::provider::SharedCredentialsProvider;
 use aws_credential_types::Credentials;
+use aws_credential_types::provider::SharedCredentialsProvider;
 use aws_smithy_async::rt::sleep::TokioSleep;
 use aws_smithy_async::time::SystemTimeSource;
 use aws_smithy_runtime_api::client::identity::{
@@ -18,7 +18,7 @@ use aws_smithy_types::config_bag::ConfigBag;
 use datafusion_common::plan_datafusion_err;
 use log::debug;
 use object_store::aws::{
-    resolve_bucket_region, AmazonS3, AmazonS3Builder, AmazonS3ConfigKey, AwsCredential,
+    AmazonS3, AmazonS3Builder, AmazonS3ConfigKey, AwsCredential, resolve_bucket_region,
 };
 use object_store::{ClientOptions, CredentialProvider};
 use tokio::sync::OnceCell;
@@ -156,21 +156,24 @@ pub fn parse_s3_url(
         )),
     })?;
     let first_path_segment = url.path_segments().into_iter().flatten().next();
-    debug!("Parsing S3 url: {url} scheme: {scheme} host: {host} first_path_segment: {first_path_segment:?}");
+    debug!(
+        "Parsing S3 url: {url} scheme: {scheme} host: {host} first_path_segment: {first_path_segment:?}"
+    );
 
     match scheme {
-        "s3" | "s3a" => {
+        "s3" | "s3a" | "oss" => {
             builder = builder.with_bucket_name(host);
-            if let Some(bucket_prefix) = host.strip_suffix("--x-s3") {
-                if let Some(_bucket_az) = bucket_prefix.rsplit_once("--") {
-                    builder = builder.with_s3_express(true);
-                }
+            if let Some(bucket_prefix) = host.strip_suffix("--x-s3")
+                && let Some(_bucket_az) = bucket_prefix.rsplit_once("--")
+            {
+                builder = builder.with_s3_express(true);
             }
         }
         "http" | "https" => {
             if scheme == "http" {
                 builder = builder.with_allow_http(true);
             }
+            let endpoint = || url[..url::Position::BeforePath].to_string();
             match host.split('.').collect::>()[..] {
                 // Support for path-style continues for buckets created on/before Sept. 30, 2020:
                 // https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/
@@ -217,6 +220,20 @@ pub fn parse_s3_url(
                         builder = builder.with_bucket_name(bucket);
                     }
                 }
+                [bucket, "s3", region, "aliyuncs", "com"] if region.starts_with("oss-") => {
+                    builder = builder.with_bucket_name(bucket);
+                    builder = builder.with_region(region.trim_start_matches("oss-"));
+                    builder = builder.with_endpoint(endpoint());
+                    builder = builder.with_virtual_hosted_style_request(true);
+                }
+                ["s3", region, "aliyuncs", "com"] if region.starts_with("oss-") => {
+                    builder = builder.with_region(region.trim_start_matches("oss-"));
+                    builder = builder.with_endpoint(endpoint());
+                    builder = builder.with_virtual_hosted_style_request(false);
+                    if let Some(bucket) = first_path_segment {
+                        builder = builder.with_bucket_name(bucket);
+                    }
+                }
                 [bucket, _s3express_zone_id, region, "amazonaws", "com"] => {
                     builder = builder.with_bucket_name(bucket);
                     builder = builder.with_region(region);
@@ -228,7 +245,7 @@ pub fn parse_s3_url(
                         source: Box::new(plan_datafusion_err!(
                             "URL did not match any known pattern for scheme: {url}"
                         )),
-                    })
+                    });
                 }
             }
         }
@@ -244,3 +261,60 @@ pub fn parse_s3_url(
 
     Ok(builder)
 }
+
+#[cfg(test)]
+#[expect(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_oss_url_sets_bucket() {
+        let url = Url::parse("oss://bucket/path/to/data").unwrap();
+        let builder = parse_s3_url(AmazonS3Builder::from_env(), &url).unwrap();
+
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Bucket),
+            Some("bucket".to_string())
+        );
+    }
+
+    #[test]
+    fn parse_aliyun_oss_virtual_hosted_endpoint() {
+        let url =
+            Url::parse("https://bucket.s3.oss-cn-hangzhou.aliyuncs.com:9443/path/to/data").unwrap();
+        let builder = parse_s3_url(AmazonS3Builder::from_env(), &url).unwrap();
+
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Bucket),
+            Some("bucket".to_string())
+        );
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Region),
+            Some("cn-hangzhou".to_string())
+        );
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Endpoint),
+            Some("https://bucket.s3.oss-cn-hangzhou.aliyuncs.com:9443".to_string())
+        );
+    }
+
+    #[test]
+    fn parse_aliyun_oss_path_style_endpoint() {
+        let url =
+            Url::parse("https://s3.oss-cn-hangzhou.aliyuncs.com:9443/bucket/path/to/data").unwrap();
+        let builder = parse_s3_url(AmazonS3Builder::from_env(), &url).unwrap();
+
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Bucket),
+            Some("bucket".to_string())
+        );
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Region),
+            Some("cn-hangzhou".to_string())
+        );
+        assert_eq!(
+            builder.get_config_value(&AmazonS3ConfigKey::Endpoint),
+            Some("https://s3.oss-cn-hangzhou.aliyuncs.com:9443".to_string())
+        );
+    }
+}
diff --git a/crates/sail-physical-optimizer/src/collect_left.rs b/crates/sail-physical-optimizer/src/collect_left.rs
index ed7fa6e7c..ee3daebfc 100644
--- a/crates/sail-physical-optimizer/src/collect_left.rs
+++ b/crates/sail-physical-optimizer/src/collect_left.rs
@@ -6,7 +6,7 @@ use datafusion::physical_optimizer::PhysicalOptimizerRule;
 use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
 use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};
 use datafusion::physical_plan::{
-    with_new_children_if_necessary, ExecutionPlan, ExecutionPlanProperties,
+    ExecutionPlan, ExecutionPlanProperties, with_new_children_if_necessary,
 };
 
 /// Safety-net rule that ensures the build side (left child) of every
@@ -131,9 +131,11 @@ mod tests {
                 .partition_count(),
             1
         );
-        assert!(new_join.children()[0]
-            .downcast_ref::()
-            .is_some());
+        assert!(
+            new_join.children()[0]
+                .downcast_ref::()
+                .is_some()
+        );
     }
 
     #[test]
@@ -167,9 +169,11 @@ mod tests {
         let result = rule.optimize(join, &config).unwrap();
 
         let new_join = result.downcast_ref::().unwrap();
-        assert!(new_join.children()[0]
-            .downcast_ref::()
-            .is_none());
+        assert!(
+            new_join.children()[0]
+                .downcast_ref::()
+                .is_none()
+        );
     }
 
     #[test]
@@ -201,8 +205,10 @@ mod tests {
         let result = rule.optimize(join, &config).unwrap();
 
         let new_join = result.downcast_ref::().unwrap();
-        assert!(new_join.children()[0]
-            .downcast_ref::()
-            .is_none());
+        assert!(
+            new_join.children()[0]
+                .downcast_ref::()
+                .is_none()
+        );
     }
 }
diff --git a/crates/sail-physical-optimizer/src/explicit_repartition.rs b/crates/sail-physical-optimizer/src/explicit_repartition.rs
index 189b30f8e..d41960ea8 100644
--- a/crates/sail-physical-optimizer/src/explicit_repartition.rs
+++ b/crates/sail-physical-optimizer/src/explicit_repartition.rs
@@ -168,9 +168,11 @@ mod tests {
 
         let optimized = optimize(plan);
 
-        assert!(optimized
-            .downcast_ref::()
-            .is_some());
+        assert!(
+            optimized
+                .downcast_ref::()
+                .is_some()
+        );
         assert_eq!(optimized.output_partitioning().partition_count(), 4);
     }
 }
diff --git a/crates/sail-physical-optimizer/src/join_reorder/builder.rs b/crates/sail-physical-optimizer/src/join_reorder/builder.rs
index 4a5a32a04..a989263fc 100644
--- a/crates/sail-physical-optimizer/src/join_reorder/builder.rs
+++ b/crates/sail-physical-optimizer/src/join_reorder/builder.rs
@@ -5,21 +5,21 @@ use datafusion::common::stats::Precision;
 use datafusion::common::tree_node::{Transformed, TreeNode};
 use datafusion::error::{DataFusionError, Result};
 use datafusion::logical_expr::{JoinType, Operator};
+use datafusion::physical_expr::PhysicalExpr;
 use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 use datafusion::physical_expr::utils::collect_columns;
-use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_plan::aggregates::AggregateExec;
 use datafusion::physical_plan::filter::FilterExec;
 use datafusion::physical_plan::joins::HashJoinExec;
 use datafusion::physical_plan::projection::ProjectionExec;
-use datafusion::physical_plan::ExecutionPlan;
 use datafusion_physical_expr::intervals::utils::check_support;
-use datafusion_physical_expr::{analyze, AnalysisContext};
+use datafusion_physical_expr::{AnalysisContext, analyze};
 use log::trace;
 
+use crate::join_reorder::JoinReorderOptions;
 use crate::join_reorder::graph::{JoinEdge, QueryGraph, RelationNode, StableColumn};
 use crate::join_reorder::join_set::JoinSet;
-use crate::join_reorder::JoinReorderOptions;
 
 type PhysicalExprRef = Arc;
 type EquiPair = (StableColumn, StableColumn);
@@ -1692,8 +1692,8 @@ mod tests {
     #[test]
     fn test_builder_keeps_true_multi_relation_predicate_as_hyperedge() -> Result<()> {
         use datafusion::common::{JoinSide, NullEquality};
-        use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};
         use datafusion::physical_plan::joins::PartitionMode;
+        use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};
 
         let mut builder = GraphBuilder::default();
 
diff --git a/crates/sail-physical-optimizer/src/join_reorder/cardinality_estimator.rs b/crates/sail-physical-optimizer/src/join_reorder/cardinality_estimator.rs
index 5520d8f01..eb4c8ae98 100644
--- a/crates/sail-physical-optimizer/src/join_reorder/cardinality_estimator.rs
+++ b/crates/sail-physical-optimizer/src/join_reorder/cardinality_estimator.rs
@@ -418,11 +418,11 @@ impl CardinalityEstimator {
 
         // Recursively count the number of base conditions in the expression
         fn count_conditions(expr: &Arc) -> usize {
-            if let Some(binary_expr) = expr.downcast_ref::() {
-                if binary_expr.op() == &Operator::And {
-                    return count_conditions(binary_expr.left())
-                        + count_conditions(binary_expr.right());
-                }
+            if let Some(binary_expr) = expr.downcast_ref::()
+                && binary_expr.op() == &Operator::And
+            {
+                return count_conditions(binary_expr.left())
+                    + count_conditions(binary_expr.right());
             }
             1 // Not an AND, count as one condition
         }
@@ -441,8 +441,8 @@ mod tests {
     use datafusion::common::stats::Precision;
     use datafusion::common::{ScalarValue, Statistics};
     use datafusion::logical_expr::{JoinType, Operator};
-    use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal};
     use datafusion::physical_expr::PhysicalExpr;
+    use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal};
     use datafusion::physical_plan::empty::EmptyExec;
 
     use super::*;
@@ -571,8 +571,8 @@ mod tests {
     #[test]
     fn test_has_non_equi_filter() {
         use datafusion::logical_expr::{JoinType, Operator};
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 
         use crate::join_reorder::graph::JoinEdge;
 
@@ -642,8 +642,8 @@ mod tests {
         use datafusion::arrow::datatypes::{DataType, Field, Schema};
         use datafusion::common::Statistics;
         use datafusion::logical_expr::JoinType;
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_plan::empty::EmptyExec;
 
         let mut graph: QueryGraph = QueryGraph::new();
@@ -728,8 +728,8 @@ mod tests {
     #[test]
     fn test_tdom_prefers_distinct_stats_over_missing_cols() -> Result<()> {
         use datafusion::logical_expr::JoinType;
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 
         let mut graph = QueryGraph::new();
         let schema: Arc =
@@ -783,8 +783,8 @@ mod tests {
     #[test]
     fn test_tdom_cap_uses_base_cardinality_for_filtered_dimension() -> Result<()> {
         use datafusion::logical_expr::JoinType;
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 
         let mut graph = QueryGraph::new();
         let schema: Arc =
@@ -853,8 +853,8 @@ mod tests {
     #[test]
     fn test_tdom_fallback_uses_min_relation_cardinality() -> Result<()> {
         use datafusion::logical_expr::JoinType;
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 
         let mut graph = QueryGraph::new();
         let schema: Arc =
@@ -915,8 +915,8 @@ mod tests {
     #[test]
     fn test_tdom_for_multi_key_edge_uses_all_equivalence_sets() -> Result<()> {
         use datafusion::logical_expr::JoinType;
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 
         let mut graph = QueryGraph::new();
         let schema: Arc = Arc::new(Schema::new(vec![
@@ -997,8 +997,8 @@ mod tests {
     #[test]
     fn test_tdom_for_multi_key_edge_is_capped_by_min_relation_cardinality() -> Result<()> {
         use datafusion::logical_expr::JoinType;
-        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
         use datafusion::physical_expr::PhysicalExpr;
+        use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 
         let mut graph = QueryGraph::new();
         let schema: Arc = Arc::new(Schema::new(vec![
diff --git a/crates/sail-physical-optimizer/src/join_reorder/cost_model.rs b/crates/sail-physical-optimizer/src/join_reorder/cost_model.rs
index ed388fc8f..480b3ccb4 100644
--- a/crates/sail-physical-optimizer/src/join_reorder/cost_model.rs
+++ b/crates/sail-physical-optimizer/src/join_reorder/cost_model.rs
@@ -1,5 +1,5 @@
-use crate::join_reorder::dp_plan::DPPlan;
 use crate::join_reorder::JoinReorderOptions;
+use crate::join_reorder::dp_plan::DPPlan;
 
 /// Cost model for evaluating the quality of a join plan.
 ///
diff --git a/crates/sail-physical-optimizer/src/join_reorder/enumerator.rs b/crates/sail-physical-optimizer/src/join_reorder/enumerator.rs
index 93d8fd138..1dfbd51cc 100644
--- a/crates/sail-physical-optimizer/src/join_reorder/enumerator.rs
+++ b/crates/sail-physical-optimizer/src/join_reorder/enumerator.rs
@@ -5,12 +5,12 @@ use datafusion::common::stats::Precision;
 use datafusion::error::{DataFusionError, Result};
 use log::{trace, warn};
 
+use crate::join_reorder::JoinReorderOptions;
 use crate::join_reorder::cardinality_estimator::CardinalityEstimator;
 use crate::join_reorder::cost_model::CostModel;
 use crate::join_reorder::dp_plan::DPPlan;
 use crate::join_reorder::graph::QueryGraph;
 use crate::join_reorder::join_set::JoinSet;
-use crate::join_reorder::JoinReorderOptions;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum JoinReorderFallbackReason {
@@ -96,16 +96,15 @@ impl PlanEnumerator {
         }
 
         // Defensive fallback: always keep at least one anchor candidate.
-        if anchor_bits == 0 {
-            if let Some(relation) = query_graph
+        if anchor_bits == 0
+            && let Some(relation) = query_graph
                 .relations
                 .iter()
                 .max_by(|left, right| left.base_cardinality.total_cmp(&right.base_cardinality))
-            {
-                anchor_bits |= 1u64 << relation.relation_id;
-                anchor_total = relation.base_cardinality.max(0.0);
-                anchor_count = 1;
-            }
+        {
+            anchor_bits |= 1u64 << relation.relation_id;
+            anchor_total = relation.base_cardinality.max(0.0);
+            anchor_count = 1;
         }
 
         let anchors = JoinSet::from_bits(anchor_bits);
@@ -760,11 +759,11 @@ mod tests {
     use std::sync::Arc;
 
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
-    use datafusion::common::stats::Precision;
     use datafusion::common::Statistics;
+    use datafusion::common::stats::Precision;
     use datafusion::logical_expr::{JoinType, Operator};
-    use datafusion::physical_expr::expressions::{BinaryExpr, Column};
     use datafusion::physical_expr::PhysicalExpr;
+    use datafusion::physical_expr::expressions::{BinaryExpr, Column};
     use datafusion::physical_plan::empty::EmptyExec;
 
     use super::*;
diff --git a/crates/sail-physical-optimizer/src/join_reorder/mod.rs b/crates/sail-physical-optimizer/src/join_reorder/mod.rs
index 174a597ab..ab78812de 100644
--- a/crates/sail-physical-optimizer/src/join_reorder/mod.rs
+++ b/crates/sail-physical-optimizer/src/join_reorder/mod.rs
@@ -8,16 +8,16 @@ use datafusion::config::ConfigOptions;
 use datafusion::error::{DataFusionError, Result};
 use datafusion::physical_expr::expressions::Column;
 use datafusion::physical_plan::projection::ProjectionExec;
-use datafusion::physical_plan::{displayable, ExecutionPlan};
+use datafusion::physical_plan::{ExecutionPlan, displayable};
 use log::{trace, warn};
 
+use crate::PhysicalOptimizerRule;
 use crate::join_reorder::builder::{ColumnMap, ColumnMapEntry, GraphBuilder};
 use crate::join_reorder::enumerator::{
     JoinReorderFallbackReason, JoinReorderStatus, PlanEnumerator,
 };
 use crate::join_reorder::graph::StableColumn;
 use crate::join_reorder::reconstructor::PlanReconstructor;
-use crate::PhysicalOptimizerRule;
 
 mod builder;
 mod cardinality_estimator;
@@ -227,8 +227,7 @@ impl JoinReorder {
             Some(plan) => {
                 trace!(
                     "JoinReorder: DP optimization completed successfully (status={:?}, emits={})",
-                    solve_result.status,
-                    emit_count
+                    solve_result.status, emit_count
                 );
                 self.record_outcome(RegionOutcome {
                     relation_count,
@@ -335,9 +334,7 @@ impl JoinReorder {
 
                     trace!(
                         "build_final_projection: Looking for stable column R{}.C{} (output_idx={})",
-                        relation_id,
-                        column_index,
-                        output_idx
+                        relation_id, column_index, output_idx
                     );
 
                     // Find this stable column's position in the final join tree output
@@ -495,9 +492,9 @@ mod tests {
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
     use datafusion::common::NullEquality;
     use datafusion::logical_expr::{JoinType, Operator};
+    use datafusion::physical_expr::PhysicalExpr;
     use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal};
     use datafusion::physical_expr::utils::collect_columns;
-    use datafusion::physical_expr::PhysicalExpr;
     use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};
     use datafusion::physical_plan::empty::EmptyExec;
     use datafusion::physical_plan::filter::FilterExec;
@@ -1060,8 +1057,8 @@ mod tests {
     // `JoinReorder::take_recorded_outcomes()` so that we catch silent regressions where the
     // rule shape is unchanged but the optimizer fell back to greedy (or vice versa).
 
-    use datafusion::common::stats::Precision;
     use datafusion::common::Statistics;
+    use datafusion::common::stats::Precision;
     use datafusion::physical_plan::test::exec::StatisticsExec;
 
     /// Build a `StatisticsExec` leaf with a fixed (id, val) schema, a known total row count,
diff --git a/crates/sail-physical-optimizer/src/join_reorder/reconstructor.rs b/crates/sail-physical-optimizer/src/join_reorder/reconstructor.rs
index 37e4bf67f..77d8b67cc 100644
--- a/crates/sail-physical-optimizer/src/join_reorder/reconstructor.rs
+++ b/crates/sail-physical-optimizer/src/join_reorder/reconstructor.rs
@@ -4,16 +4,16 @@ use std::sync::Arc;
 use datafusion::common::{JoinSide, NullEquality};
 use datafusion::error::{DataFusionError, Result};
 use datafusion::logical_expr::{JoinType, Operator};
+use datafusion::physical_expr::PhysicalExpr;
 use datafusion::physical_expr::expressions::{BinaryExpr, Column};
 use datafusion::physical_expr::utils::collect_columns;
-use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_plan::filter::FilterExec;
 use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter};
 use datafusion::physical_plan::joins::{
     CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode,
 };
 use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
-use datafusion::physical_plan::ExecutionPlan;
 use log::warn;
 
 use crate::join_reorder::builder::{ColumnMap, ColumnMapEntry};
@@ -900,7 +900,10 @@ impl<'a> PlanReconstructor<'a> {
                 )));
             }
 
-            match (left_matches.first().copied(), right_matches.first().copied()) {
+            match (
+                left_matches.first().copied(),
+                right_matches.first().copied(),
+            ) {
                 (Some(_), Some(_)) => Err(DataFusionError::Internal(format!(
                     "Ambiguous column reference '{}' found in both left and right join inputs during reconstruction",
                     col.name()
@@ -1023,8 +1026,8 @@ impl<'a> PlanReconstructor<'a> {
         let transformed = expr_arc.transform(|node| {
             if let Some(col) = node.downcast_ref::() {
                 // Prefer stable name mapping first
-                if let Some((rel, cidx)) = StableColumn::parse_stable_name(col.name()) {
-                    if let Some(pos) = output_map.iter().position(|e| {
+                if let Some((rel, cidx)) = StableColumn::parse_stable_name(col.name())
+                    && let Some(pos) = output_map.iter().position(|e| {
                         matches!(
                             e,
                             ColumnMapEntry::Stable {
@@ -1032,12 +1035,12 @@ impl<'a> PlanReconstructor<'a> {
                                 column_index
                             } if *relation_id == rel && *column_index == cidx
                         )
-                    }) {
-                        // Use the final plan's schema field name and index
-                        let field_name = plan.schema().field(pos).name().to_string();
-                        let new_col = Column::new(&field_name, pos);
-                        return Ok(Transformed::yes(Arc::new(new_col)));
-                    }
+                    })
+                {
+                    // Use the final plan's schema field name and index
+                    let field_name = plan.schema().field(pos).name().to_string();
+                    let new_col = Column::new(&field_name, pos);
+                    return Ok(Transformed::yes(Arc::new(new_col)));
                 }
 
                 // Fallback: try to match by current schema field name.
diff --git a/crates/sail-physical-optimizer/src/lib.rs b/crates/sail-physical-optimizer/src/lib.rs
index 662564992..3dfef7ea2 100644
--- a/crates/sail-physical-optimizer/src/lib.rs
+++ b/crates/sail-physical-optimizer/src/lib.rs
@@ -1,5 +1,6 @@
 use std::sync::Arc;
 
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
 use datafusion::physical_optimizer::aggregate_statistics::AggregateStatistics;
 use datafusion::physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;
 use datafusion::physical_optimizer::enforce_distribution::EnforceDistribution;
@@ -19,7 +20,6 @@ use datafusion::physical_optimizer::topk_aggregation::TopKAggregation;
 use datafusion::physical_optimizer::topk_repartition::TopKRepartition;
 use datafusion::physical_optimizer::update_aggr_exprs::OptimizeAggregateOrder;
 use datafusion::physical_optimizer::window_topn::WindowTopN;
-use datafusion::physical_optimizer::PhysicalOptimizerRule;
 
 use crate::barrier::EnforceBarrierPartitioning;
 use crate::collect_left::RewriteCollectLeftHashJoin;
@@ -94,8 +94,7 @@ mod tests {
             .filter(|name| datafusion_optimizer_names.contains(name))
             .collect();
         assert_eq!(
-            datafusion_optimizer_names,
-            actual_datafusion_optimizer_names,
+            datafusion_optimizer_names, actual_datafusion_optimizer_names,
             "the custom physical optimizer rules should include all the default DataFusion optimizer rules in the same order"
         );
 
diff --git a/crates/sail-physical-plan/Cargo.toml b/crates/sail-physical-plan/Cargo.toml
index 5a76c127c..668f64bc0 100644
--- a/crates/sail-physical-plan/Cargo.toml
+++ b/crates/sail-physical-plan/Cargo.toml
@@ -8,13 +8,21 @@ workspace = true
 
 [dependencies]
 sail-catalog = { path = "../sail-catalog" }
+sail-cache = { path = "../sail-cache" }
 sail-common-datafusion = { path = "../sail-common-datafusion" }
 sail-logical-plan = { path = "../sail-logical-plan" }
 
 datafusion = { workspace = true }
 datafusion-common = { workspace = true }
+datafusion-datasource-parquet = { workspace = true }
 datafusion-expr = { workspace = true }
 futures = { workspace = true }
+object_store = { workspace = true }
+parquet = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
 lazy_static = { workspace = true }
+log = { workspace = true }
+uuid = { workspace = true }
diff --git a/crates/sail-physical-plan/src/barrier.rs b/crates/sail-physical-plan/src/barrier.rs
index c5052248d..deafdbd80 100644
--- a/crates/sail-physical-plan/src/barrier.rs
+++ b/crates/sail-physical-plan/src/barrier.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::execution::{SendableRecordBatchStream, TaskContext};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
-use datafusion_common::{exec_err, Result};
+use datafusion_common::{Result, exec_err};
 use futures::{StreamExt, TryStreamExt};
 
 /// A physical plan node that enforces a barrier between preconditions and the actual plan.
diff --git a/crates/sail-physical-plan/src/catalog_command.rs b/crates/sail-physical-plan/src/catalog_command.rs
index 8d3a67633..aefca554d 100644
--- a/crates/sail-physical-plan/src/catalog_command.rs
+++ b/crates/sail-physical-plan/src/catalog_command.rs
@@ -6,7 +6,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning};
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
-use datafusion_common::{exec_err, internal_err, Result};
+use datafusion_common::{Result, exec_err, internal_err};
 use sail_catalog::command::CatalogCommand;
 use sail_catalog::manager::CatalogManager;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
diff --git a/crates/sail-physical-plan/src/coalesce.rs b/crates/sail-physical-plan/src/coalesce.rs
index 3aa56f528..8d1ccc626 100644
--- a/crates/sail-physical-plan/src/coalesce.rs
+++ b/crates/sail-physical-plan/src/coalesce.rs
@@ -14,7 +14,7 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
     RecordBatchStream,
 };
-use datafusion_common::{internal_err, plan_err, Result, Statistics};
+use datafusion_common::{Result, Statistics, internal_err, plan_err};
 use futures::Stream;
 
 /// A physical plan node for Spark-style coalesce without a shuffle.
@@ -202,7 +202,7 @@ mod tests {
     use datafusion_common::Result;
     use futures::StreamExt;
 
-    use super::{coalesced_input_range, CoalesceExec};
+    use super::{CoalesceExec, coalesced_input_range};
 
     fn test_plan(partitions: Vec>) -> Arc {
         let schema = if let Some(batch) = partitions.iter().flatten().next() {
diff --git a/crates/sail-physical-plan/src/data_source.rs b/crates/sail-physical-plan/src/data_source.rs
new file mode 100644
index 000000000..cb6d4a91f
--- /dev/null
+++ b/crates/sail-physical-plan/src/data_source.rs
@@ -0,0 +1,71 @@
+use std::fmt;
+use std::sync::Arc;
+
+use datafusion::datasource::source::{DataSource, DataSourceExec};
+use datafusion::execution::{SendableRecordBatchStream, TaskContext};
+use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
+use datafusion_common::{Result, exec_err, internal_err};
+
+/// A [`DataSourceExec`] wrapper for Sail remote codec.
+///
+/// DataFusion's built-in physical plan codec can currently mishandle file
+/// compression metadata for data sources in cluster mode. We wrap
+/// [`DataSourceExec`] before remote plan encoding so our extension codec handles
+/// every supported data source explicitly. The wrapper is not present
+/// in decoded remote plans so it would never be executed.
+#[derive(Clone, Debug)]
+pub struct RemoteDataSourceExec {
+    data_source: Arc,
+    properties: Arc,
+}
+
+impl RemoteDataSourceExec {
+    pub fn new(node: &DataSourceExec) -> Self {
+        Self {
+            data_source: Arc::clone(node.data_source()),
+            properties: Arc::clone(node.properties()),
+        }
+    }
+
+    pub fn data_source(&self) -> &Arc {
+        &self.data_source
+    }
+}
+
+impl DisplayAs for RemoteDataSourceExec {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.name())
+    }
+}
+
+impl ExecutionPlan for RemoteDataSourceExec {
+    fn name(&self) -> &str {
+        Self::static_name()
+    }
+
+    fn properties(&self) -> &Arc {
+        &self.properties
+    }
+
+    fn children(&self) -> Vec<&Arc> {
+        Vec::new()
+    }
+
+    fn with_new_children(
+        self: Arc,
+        children: Vec>,
+    ) -> Result> {
+        if !children.is_empty() {
+            return internal_err!("{} should not have children", self.name());
+        }
+        Ok(self)
+    }
+
+    fn execute(
+        &self,
+        _partition: usize,
+        _context: Arc,
+    ) -> Result {
+        exec_err!("{} cannot be executed", self.name())
+    }
+}
diff --git a/crates/sail-physical-plan/src/lib.rs b/crates/sail-physical-plan/src/lib.rs
index 2c4ef7d0c..c5c5cbaa1 100644
--- a/crates/sail-physical-plan/src/lib.rs
+++ b/crates/sail-physical-plan/src/lib.rs
@@ -1,10 +1,12 @@
 pub mod barrier;
 pub mod catalog_command;
 pub mod coalesce;
+pub mod data_source;
 pub mod map_partitions;
 pub mod merge_cardinality_check;
 pub mod monotonic_id;
 pub mod range;
+pub mod remote_checkpoint;
 pub mod repartition;
 pub mod schema_pivot;
 pub mod show_string;
diff --git a/crates/sail-physical-plan/src/map_partitions.rs b/crates/sail-physical-plan/src/map_partitions.rs
index 76f9fc8c5..5a40d33ef 100644
--- a/crates/sail-physical-plan/src/map_partitions.rs
+++ b/crates/sail-physical-plan/src/map_partitions.rs
@@ -7,7 +7,7 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
     DisplayAs, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use datafusion_common::{internal_datafusion_err, Result};
+use datafusion_common::{Result, internal_datafusion_err};
 use sail_common_datafusion::array::record_batch::record_batch_with_schema;
 use sail_common_datafusion::udf::StreamUDF;
 use sail_common_datafusion::utils::items::ItemTaker;
diff --git a/crates/sail-physical-plan/src/merge_cardinality_check.rs b/crates/sail-physical-plan/src/merge_cardinality_check.rs
index 84236ba85..d618f7aa3 100644
--- a/crates/sail-physical-plan/src/merge_cardinality_check.rs
+++ b/crates/sail-physical-plan/src/merge_cardinality_check.rs
@@ -10,7 +10,7 @@ use datafusion::execution::{SendableRecordBatchStream, TaskContext};
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream,
 };
-use datafusion_common::{internal_err, DataFusionError, Result, Statistics};
+use datafusion_common::{DataFusionError, Result, Statistics, internal_err};
 use futures::Stream;
 
 #[derive(Debug)]
@@ -230,12 +230,10 @@ impl Stream for MergeCardinalityCheckStream {
                     }
                     let id = row_id_values.key(i);
                     if !self.seen.insert(id.clone()) {
-                        return Poll::Ready(Some(Err(DataFusionError::Execution(
-                            format!(
-                                "MERGE_CARDINALITY_VIOLATION: Multiple source rows matched target row '{}'",
-                                id
-                            ),
-                        ))));
+                        return Poll::Ready(Some(Err(DataFusionError::Execution(format!(
+                            "MERGE_CARDINALITY_VIOLATION: Multiple source rows matched target row '{}'",
+                            id
+                        )))));
                     }
                 }
 
diff --git a/crates/sail-physical-plan/src/monotonic_id.rs b/crates/sail-physical-plan/src/monotonic_id.rs
index 0e749f0ef..d28ea20ab 100644
--- a/crates/sail-physical-plan/src/monotonic_id.rs
+++ b/crates/sail-physical-plan/src/monotonic_id.rs
@@ -12,7 +12,7 @@ use datafusion::physical_plan::{
     RecordBatchStream,
 };
 use datafusion_common::stats::Precision;
-use datafusion_common::{exec_err, internal_err, ColumnStatistics, Result, Statistics};
+use datafusion_common::{ColumnStatistics, Result, Statistics, exec_err, internal_err};
 use futures::Stream;
 
 #[derive(Debug, Clone)]
@@ -42,7 +42,8 @@ impl MonotonicIdExec {
         }
 
         let properties = Arc::new(PlanProperties::new(
-            EquivalenceProperties::new(schema.clone()),
+            EquivalenceProperties::new(schema.clone())
+                .extend(input.equivalence_properties().clone())?,
             input.output_partitioning().clone(),
             input.pipeline_behavior(),
             input.boundedness(),
diff --git a/crates/sail-physical-plan/src/range.rs b/crates/sail-physical-plan/src/range.rs
index 47159af5a..71f02ba08 100644
--- a/crates/sail-physical-plan/src/range.rs
+++ b/crates/sail-physical-plan/src/range.rs
@@ -8,7 +8,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning};
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{DisplayAs, ExecutionPlan, PlanProperties};
-use datafusion_common::{exec_err, internal_err, plan_err, Result};
+use datafusion_common::{Result, exec_err, internal_err, plan_err};
 use sail_logical_plan::range::Range;
 
 const RANGE_BATCH_SIZE: usize = 1024;
diff --git a/crates/sail-physical-plan/src/remote_checkpoint.rs b/crates/sail-physical-plan/src/remote_checkpoint.rs
new file mode 100644
index 000000000..f693f9a7f
--- /dev/null
+++ b/crates/sail-physical-plan/src/remote_checkpoint.rs
@@ -0,0 +1,1052 @@
+use std::any::Any;
+use std::collections::BTreeMap;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringArray};
+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::common::config::{ConfigOptions, TableParquetOptions};
+use datafusion::datasource::listing::ListingTableUrl;
+use datafusion::datasource::physical_plan::{FileOutputMode, FileSinkConfig};
+use datafusion::datasource::sink::DataSink;
+use datafusion::datasource::source::{DataSource, OpenArgs};
+use datafusion::execution::object_store::ObjectStoreUrl;
+use datafusion::execution::{SendableRecordBatchStream, TaskContext};
+use datafusion::logical_expr::dml::InsertOp;
+use datafusion::physical_expr::projection::ProjectionExprs;
+use datafusion::physical_expr::{
+    Distribution, EquivalenceProperties, LexOrdering, Partitioning, PhysicalExpr, PhysicalSortExpr,
+};
+use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, SchedulingType};
+use datafusion::physical_plan::filter_pushdown::FilterPushdownPropagation;
+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
+    SortOrderPushdownResult,
+};
+use datafusion_common::{DataFusionError, Result, Statistics, internal_datafusion_err};
+use datafusion_datasource_parquet::ParquetSink;
+use futures::{StreamExt, stream};
+use object_store::path::Path;
+use object_store::{ObjectStore, ObjectStoreExt};
+use sail_cache::remote_checkpoint::{
+    RemoteCheckpointDescriptor, RemoteCheckpointFile, RemoteCheckpointPartition,
+    RemoteCheckpointRegistry,
+};
+use sail_common_datafusion::array::record_batch::record_batch_with_schema;
+use sail_common_datafusion::extension::SessionExtensionAccessor;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+pub const METADATA_COLUMN: &str = "metadata";
+
+/// Exposes the partitioning recorded by a checkpoint instead of the layout-derived partitioning
+/// reported by its underlying file or memory source.
+///
+/// Source rewrites remain wrapped so optimizer pushdowns cannot discard this property.
+#[derive(Debug)]
+pub struct CheckpointDataSource {
+    source: Arc,
+    output_partitioning: Partitioning,
+}
+
+impl CheckpointDataSource {
+    pub fn new(source: Arc, output_partitioning: Partitioning) -> Self {
+        Self {
+            source,
+            output_partitioning,
+        }
+    }
+
+    pub fn source(&self) -> &Arc {
+        &self.source
+    }
+
+    pub fn checkpoint_partitioning(&self) -> &Partitioning {
+        &self.output_partitioning
+    }
+
+    fn map_source(&self, source: Arc) -> Arc {
+        Arc::new(Self::new(source, self.output_partitioning.clone()))
+    }
+}
+
+impl DataSource for CheckpointDataSource {
+    fn open(
+        &self,
+        partition: usize,
+        context: Arc,
+    ) -> Result {
+        self.source.open(partition, context)
+    }
+
+    fn fmt_as(&self, display_type: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+        self.source.fmt_as(display_type, f)
+    }
+
+    fn output_partitioning(&self) -> Partitioning {
+        self.output_partitioning.clone()
+    }
+
+    fn eq_properties(&self) -> EquivalenceProperties {
+        self.source.eq_properties()
+    }
+
+    fn scheduling_type(&self) -> SchedulingType {
+        self.source.scheduling_type()
+    }
+
+    fn partition_statistics(&self, partition: Option) -> Result> {
+        self.source.partition_statistics(partition)
+    }
+
+    fn with_fetch(&self, limit: Option) -> Option> {
+        self.source
+            .with_fetch(limit)
+            .map(|source| self.map_source(source))
+    }
+
+    fn fetch(&self) -> Option {
+        self.source.fetch()
+    }
+
+    fn metrics(&self) -> ExecutionPlanMetricsSet {
+        self.source.metrics()
+    }
+
+    fn try_swapping_with_projection(
+        &self,
+        _projection: &ProjectionExprs,
+    ) -> Result>> {
+        Ok(None)
+    }
+
+    fn try_pushdown_filters(
+        &self,
+        filters: Vec>,
+        config: &ConfigOptions,
+    ) -> Result>> {
+        let result = self.source.try_pushdown_filters(filters, config)?;
+        Ok(FilterPushdownPropagation {
+            filters: result.filters,
+            updated_node: result.updated_node.map(|source| self.map_source(source)),
+        })
+    }
+
+    fn try_pushdown_sort(
+        &self,
+        order: &[PhysicalSortExpr],
+    ) -> Result>> {
+        Ok(self
+            .source
+            .try_pushdown_sort(order)?
+            .map(|source| self.map_source(source)))
+    }
+
+    fn with_preserve_order(&self, preserve_order: bool) -> Option> {
+        self.source
+            .with_preserve_order(preserve_order)
+            .map(|source| self.map_source(source))
+    }
+
+    fn with_new_state(&self, state: Arc) -> Option> {
+        self.source
+            .with_new_state(state)
+            .map(|source| self.map_source(source))
+    }
+
+    fn create_sibling_state(&self, config: &ConfigOptions) -> Option> {
+        self.source.create_sibling_state(config)
+    }
+
+    fn open_with_args(&self, args: OpenArgs) -> Result {
+        self.source.open_with_args(args)
+    }
+}
+
+pub fn checkpoint_storage_schema(logical_schema: &SchemaRef) -> SchemaRef {
+    let fields = logical_schema
+        .fields()
+        .iter()
+        .enumerate()
+        .map(|(index, field)| Arc::new(field.as_ref().clone().with_name(format!("_c{index}"))))
+        .collect::>();
+    Arc::new(Schema::new_with_metadata(
+        fields,
+        logical_schema.metadata().clone(),
+    ))
+}
+
+fn checkpoint_metadata_schema() -> SchemaRef {
+    Arc::new(Schema::new(vec![Field::new(
+        METADATA_COLUMN,
+        DataType::Utf8,
+        false,
+    )]))
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct CheckpointPartitionMetadata {
+    partition: u64,
+    location: Option,
+    size: u64,
+    row_count: u64,
+}
+
+fn checkpoint_command_schema() -> SchemaRef {
+    Arc::new(Schema::empty())
+}
+
+#[derive(Debug)]
+pub struct RemoteCheckpointCommitExec {
+    input: Arc,
+    relation_id: String,
+    object_store_url: ObjectStoreUrl,
+    prefix: Path,
+    logical_schema: SchemaRef,
+    storage_schema: SchemaRef,
+    output_partitioning: Partitioning,
+    output_ordering: Option,
+    properties: Arc,
+}
+
+impl RemoteCheckpointCommitExec {
+    pub fn new(
+        input: Arc,
+        relation_id: String,
+        object_store_url: ObjectStoreUrl,
+        prefix: Path,
+        logical_schema: SchemaRef,
+        storage_schema: SchemaRef,
+        output_partitioning: Partitioning,
+        output_ordering: Option,
+    ) -> Self {
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(checkpoint_command_schema()),
+            Partitioning::UnknownPartitioning(1),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        Self {
+            input,
+            relation_id,
+            object_store_url,
+            prefix,
+            logical_schema,
+            storage_schema,
+            output_partitioning,
+            output_ordering,
+            properties,
+        }
+    }
+
+    pub fn input(&self) -> &Arc {
+        &self.input
+    }
+
+    pub fn relation_id(&self) -> &str {
+        &self.relation_id
+    }
+
+    pub fn object_store_url(&self) -> &ObjectStoreUrl {
+        &self.object_store_url
+    }
+
+    pub fn prefix(&self) -> &Path {
+        &self.prefix
+    }
+
+    pub fn logical_schema(&self) -> &SchemaRef {
+        &self.logical_schema
+    }
+
+    pub fn storage_schema(&self) -> &SchemaRef {
+        &self.storage_schema
+    }
+
+    pub fn checkpoint_partitioning(&self) -> &Partitioning {
+        &self.output_partitioning
+    }
+
+    pub fn checkpoint_ordering(&self) -> Option<&LexOrdering> {
+        self.output_ordering.as_ref()
+    }
+}
+
+impl DisplayAs for RemoteCheckpointCommitExec {
+    fn fmt_as(&self, _display_type: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "RemoteCheckpointCommitExec: relation_id={}",
+            self.relation_id
+        )
+    }
+}
+
+impl ExecutionPlan for RemoteCheckpointCommitExec {
+    fn name(&self) -> &str {
+        Self::static_name()
+    }
+
+    fn properties(&self) -> &Arc {
+        &self.properties
+    }
+
+    fn required_input_distribution(&self) -> Vec {
+        vec![Distribution::SinglePartition]
+    }
+
+    fn benefits_from_input_partitioning(&self) -> Vec {
+        vec![false]
+    }
+
+    fn children(&self) -> Vec<&Arc> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc,
+        children: Vec>,
+    ) -> Result> {
+        let [input] = children.as_slice() else {
+            return Err(internal_datafusion_err!(
+                "RemoteCheckpointCommitExec must have exactly one child"
+            ));
+        };
+        Ok(Arc::new(Self::new(
+            Arc::clone(input),
+            self.relation_id.clone(),
+            self.object_store_url.clone(),
+            self.prefix.clone(),
+            Arc::clone(&self.logical_schema),
+            Arc::clone(&self.storage_schema),
+            self.output_partitioning.clone(),
+            self.output_ordering.clone(),
+        )))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc,
+    ) -> Result {
+        if partition != 0 {
+            return Err(internal_datafusion_err!(
+                "RemoteCheckpointCommitExec can only execute partition 0"
+            ));
+        }
+        let input_partitions = self.input.output_partitioning().partition_count();
+        if input_partitions != 1 {
+            return Err(internal_datafusion_err!(
+                "RemoteCheckpointCommitExec requires one input partition, got {input_partitions}"
+            ));
+        }
+        let input = self.input.execute(0, Arc::clone(&context))?;
+        let relation_id = self.relation_id.clone();
+        let object_store_url = self.object_store_url.clone();
+        let prefix = self.prefix.clone();
+        let logical_schema = Arc::clone(&self.logical_schema);
+        let storage_schema = Arc::clone(&self.storage_schema);
+        let output_partitioning = self.output_partitioning.clone();
+        let output_ordering = self.output_ordering.clone();
+        let output_schema = self.schema();
+        let stream_schema = Arc::clone(&output_schema);
+        let output = stream::once(async move {
+            commit_checkpoint(
+                input,
+                context,
+                relation_id,
+                object_store_url,
+                prefix,
+                logical_schema,
+                storage_schema,
+                output_partitioning,
+                output_ordering,
+                output_schema,
+            )
+            .await
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            stream_schema,
+            output,
+        )))
+    }
+}
+
+#[derive(Debug)]
+pub struct RemoteCheckpointWriteExec {
+    input: Arc,
+    object_store_url: ObjectStoreUrl,
+    prefix: Path,
+    storage_schema: SchemaRef,
+    properties: Arc,
+}
+
+impl RemoteCheckpointWriteExec {
+    pub fn try_new(
+        input: Arc,
+        object_store_url: ObjectStoreUrl,
+        prefix: Path,
+        storage_schema: SchemaRef,
+    ) -> Result {
+        if matches!(input.boundedness(), Boundedness::Unbounded { .. }) {
+            return Err(DataFusionError::NotImplemented(
+                "checkpoint does not support unbounded input".to_string(),
+            ));
+        }
+        let input_schema = input.schema();
+        let schema_matches = input_schema.fields().len() == storage_schema.fields().len()
+            && input_schema
+                .fields()
+                .iter()
+                .zip(storage_schema.fields())
+                .all(|(input, storage)| input.data_type() == storage.data_type());
+        if !schema_matches {
+            return Err(DataFusionError::Plan(
+                "checkpoint storage schema does not match the input schema by position".to_string(),
+            ));
+        }
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(checkpoint_metadata_schema()),
+            Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()),
+            EmissionType::Final,
+            Boundedness::Bounded,
+        ));
+        Ok(Self {
+            input,
+            object_store_url,
+            prefix,
+            storage_schema,
+            properties,
+        })
+    }
+
+    pub fn input(&self) -> &Arc {
+        &self.input
+    }
+
+    pub fn object_store_url(&self) -> &ObjectStoreUrl {
+        &self.object_store_url
+    }
+
+    pub fn prefix(&self) -> &Path {
+        &self.prefix
+    }
+
+    pub fn storage_schema(&self) -> &SchemaRef {
+        &self.storage_schema
+    }
+}
+
+impl DisplayAs for RemoteCheckpointWriteExec {
+    fn fmt_as(&self, _display_type: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "RemoteCheckpointWriteExec: object_store_url={}, prefix={}",
+            self.object_store_url, self.prefix
+        )
+    }
+}
+
+impl ExecutionPlan for RemoteCheckpointWriteExec {
+    fn name(&self) -> &str {
+        Self::static_name()
+    }
+
+    fn properties(&self) -> &Arc {
+        &self.properties
+    }
+
+    fn benefits_from_input_partitioning(&self) -> Vec {
+        vec![false]
+    }
+
+    fn children(&self) -> Vec<&Arc> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc,
+        children: Vec>,
+    ) -> Result> {
+        let [input] = children.as_slice() else {
+            return Err(internal_datafusion_err!(
+                "RemoteCheckpointWriteExec must have exactly one child"
+            ));
+        };
+        Ok(Arc::new(Self::try_new(
+            Arc::clone(input),
+            self.object_store_url.clone(),
+            self.prefix.clone(),
+            Arc::clone(&self.storage_schema),
+        )?))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc,
+    ) -> Result {
+        let input = self.input.execute(partition, Arc::clone(&context))?;
+        let store = context.runtime_env().object_store(&self.object_store_url)?;
+        let location = self
+            .prefix
+            .clone()
+            .join(format!("part-{partition:05}-{}.parquet", Uuid::new_v4()));
+        let storage_schema = Arc::clone(&self.storage_schema);
+        let output_schema = self.schema();
+        let stream_schema = Arc::clone(&output_schema);
+        let output = stream::once(write_checkpoint_partition(
+            partition,
+            input,
+            context,
+            self.object_store_url.clone(),
+            store,
+            location,
+            storage_schema,
+            output_schema,
+        ));
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            stream_schema,
+            output,
+        )))
+    }
+}
+
+async fn write_checkpoint_partition(
+    partition: usize,
+    mut input: SendableRecordBatchStream,
+    context: Arc,
+    object_store_url: ObjectStoreUrl,
+    store: Arc,
+    location: Path,
+    storage_schema: SchemaRef,
+    output_schema: SchemaRef,
+) -> Result {
+    let mut row_count = 0_u64;
+    let first_batch = loop {
+        let Some(batch) = input.next().await else {
+            break None;
+        };
+        let batch = batch?;
+        row_count = row_count
+            .checked_add(u64::try_from(batch.num_rows()).map_err(|_| {
+                internal_datafusion_err!("checkpoint partition row count is too large")
+            })?)
+            .ok_or_else(|| internal_datafusion_err!("checkpoint partition row count overflow"))?;
+        if storage_schema.fields().is_empty() || batch.num_rows() == 0 {
+            continue;
+        }
+        break Some(batch);
+    };
+
+    // An empty partition stays in the descriptor but does not allocate an object.
+    let (location, size, row_count) = if let Some(first_batch) = first_batch {
+        let stream_schema = Arc::clone(&storage_schema);
+        let batch_schema = Arc::clone(&storage_schema);
+        let batches = stream::once(async move { Ok(first_batch) })
+            .chain(input)
+            .map(move |batch| {
+                batch.and_then(|batch| record_batch_with_schema(batch, &batch_schema))
+            });
+        let batches: SendableRecordBatchStream =
+            Box::pin(RecordBatchStreamAdapter::new(stream_schema, batches));
+        let location_url = format!("{object_store_url}{location}");
+        let sink = ParquetSink::new(
+            FileSinkConfig {
+                original_url: location_url.clone(),
+                object_store_url,
+                file_group: Default::default(),
+                table_paths: vec![ListingTableUrl::parse(location_url)?],
+                output_schema: storage_schema,
+                table_partition_cols: vec![],
+                insert_op: InsertOp::Append,
+                keep_partition_by_columns: false,
+                file_extension: "parquet".to_string(),
+                file_output_mode: FileOutputMode::SingleFile,
+            },
+            TableParquetOptions {
+                global: context.session_config().options().execution.parquet.clone(),
+                ..Default::default()
+            },
+        );
+        let row_count = sink.write_all(batches, &context).await?;
+        let size = store
+            .head(&location)
+            .await
+            .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?
+            .size;
+        (Some(location.to_string()), size, row_count)
+    } else {
+        (None, 0, row_count)
+    };
+    let partition = u64::try_from(partition)
+        .map_err(|_| internal_datafusion_err!("checkpoint partition index is too large"))?;
+    let metadata = serde_json::to_string(&CheckpointPartitionMetadata {
+        partition,
+        location,
+        size,
+        row_count,
+    })
+    .map_err(|error| internal_datafusion_err!("failed to encode checkpoint metadata: {error}"))?;
+    let columns: Vec = vec![Arc::new(StringArray::from(vec![metadata]))];
+    Ok(RecordBatch::try_new(output_schema, columns)?)
+}
+
+#[expect(clippy::too_many_arguments)]
+async fn commit_checkpoint(
+    input: SendableRecordBatchStream,
+    context: Arc,
+    relation_id: String,
+    object_store_url: ObjectStoreUrl,
+    prefix: Path,
+    logical_schema: SchemaRef,
+    storage_schema: SchemaRef,
+    output_partitioning: Partitioning,
+    output_ordering: Option,
+    output_schema: SchemaRef,
+) -> Result {
+    let registry = context.extension::()?;
+    let commit = async {
+        let partitions = collect_checkpoint_partitions(input, &prefix, &storage_schema).await?;
+        let output_partitioning =
+            checkpoint_partitioning_with_count(output_partitioning, partitions.len());
+        registry.insert(RemoteCheckpointDescriptor {
+            relation_id,
+            object_store_url: object_store_url.clone(),
+            prefix: prefix.clone(),
+            logical_schema,
+            storage_schema,
+            output_partitioning,
+            output_ordering,
+            partitions,
+        })?;
+        Ok(RecordBatch::new_empty(output_schema))
+    }
+    .await;
+    match commit {
+        Ok(batch) => Ok(batch),
+        Err(error) => {
+            match registry
+                .cleanup_relation(context.runtime_env().as_ref(), &object_store_url, &prefix)
+                .await
+            {
+                Ok(()) => Err(error),
+                Err(cleanup_error) => Err(internal_datafusion_err!(
+                    "checkpoint failed: {error}; additionally failed to clean {prefix}: {cleanup_error}"
+                )),
+            }
+        }
+    }
+}
+
+async fn collect_checkpoint_partitions(
+    mut stream: SendableRecordBatchStream,
+    relation_prefix: &Path,
+    storage_schema: &Schema,
+) -> Result> {
+    let mut partitions = BTreeMap::new();
+    while let Some(batch) = stream.next().await {
+        let batch = batch?;
+        if batch.num_columns() != 1 {
+            return Err(internal_datafusion_err!(
+                "checkpoint returned {} metadata columns instead of 1",
+                batch.num_columns()
+            ));
+        }
+        let metadata_values = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| {
+                internal_datafusion_err!("invalid checkpoint {METADATA_COLUMN} column")
+            })?;
+        for row in 0..batch.num_rows() {
+            if metadata_values.is_null(row) {
+                return Err(internal_datafusion_err!(
+                    "checkpoint returned null metadata"
+                ));
+            }
+            let CheckpointPartitionMetadata {
+                partition,
+                location,
+                size,
+                row_count,
+            } = serde_json::from_str(metadata_values.value(row)).map_err(|error| {
+                internal_datafusion_err!("invalid checkpoint metadata JSON: {error}")
+            })?;
+            let partition = usize::try_from(partition)
+                .map_err(|_| internal_datafusion_err!("checkpoint partition index is too large"))?;
+            let file = if let Some(location) = location {
+                if storage_schema.fields().is_empty() {
+                    return Err(internal_datafusion_err!(
+                        "zero-column checkpoint partition {partition} returned a file"
+                    ));
+                }
+                if size == 0 || row_count == 0 {
+                    return Err(internal_datafusion_err!(
+                        "checkpoint partition {partition} has an empty file"
+                    ));
+                }
+                let location = Path::parse(location).map_err(|error| {
+                    internal_datafusion_err!("invalid checkpoint object location: {error}")
+                })?;
+                validate_partition_location(relation_prefix, partition, &location)?;
+                Some(RemoteCheckpointFile { location, size })
+            } else {
+                if size != 0 {
+                    return Err(internal_datafusion_err!(
+                        "checkpoint partition {partition} has a size without a file"
+                    ));
+                }
+                if row_count != 0 && !storage_schema.fields().is_empty() {
+                    return Err(internal_datafusion_err!(
+                        "non-empty checkpoint partition {partition} is missing a file"
+                    ));
+                }
+                None
+            };
+            if partitions
+                .insert(
+                    partition,
+                    RemoteCheckpointPartition {
+                        partition,
+                        row_count,
+                        file,
+                    },
+                )
+                .is_some()
+            {
+                return Err(internal_datafusion_err!(
+                    "checkpoint returned duplicate partition {partition}"
+                ));
+            }
+        }
+    }
+    if partitions.is_empty() {
+        return Err(internal_datafusion_err!(
+            "checkpoint did not return partition 0"
+        ));
+    }
+    partitions
+        .into_iter()
+        .enumerate()
+        .map(|(expected, (partition, value))| {
+            if partition != expected {
+                return Err(internal_datafusion_err!(
+                    "checkpoint did not return partition {expected}"
+                ));
+            }
+            Ok(value)
+        })
+        .collect()
+}
+
+fn checkpoint_partitioning_with_count(
+    partitioning: Partitioning,
+    partition_count: usize,
+) -> Partitioning {
+    match partitioning {
+        Partitioning::RoundRobinBatch(_) => Partitioning::RoundRobinBatch(partition_count),
+        Partitioning::Hash(expressions, expected) if expected == partition_count => {
+            Partitioning::Hash(expressions, partition_count)
+        }
+        Partitioning::Hash(_, _) => Partitioning::UnknownPartitioning(partition_count),
+        Partitioning::UnknownPartitioning(_) => Partitioning::UnknownPartitioning(partition_count),
+    }
+}
+
+fn validate_partition_location(
+    relation_prefix: &Path,
+    partition: usize,
+    location: &Path,
+) -> Result<()> {
+    let Some(filename) = location.filename() else {
+        return Err(internal_datafusion_err!(
+            "checkpoint returned an invalid object for partition {partition}: {location}"
+        ));
+    };
+    let prefix = format!("part-{partition:05}-");
+    let file_id = filename
+        .strip_prefix(&prefix)
+        .and_then(|value| value.strip_suffix(".parquet"));
+    let valid_file_id = file_id.is_some_and(|value| Uuid::parse_str(value).is_ok());
+    if !location.prefix_matches(relation_prefix)
+        || location.parts_count() != relation_prefix.parts_count() + 1
+        || !valid_file_id
+    {
+        return Err(internal_datafusion_err!(
+            "checkpoint returned unexpected object {location} for partition {partition}"
+        ));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+#[expect(clippy::expect_used)]
+mod tests {
+    use datafusion::arrow::array::Int32Array;
+    use datafusion::arrow::record_batch::RecordBatchOptions;
+    use datafusion::datasource::memory::MemorySourceConfig;
+    use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+    use datafusion::prelude::{SessionConfig, SessionContext};
+    use object_store::memory::InMemory;
+    use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+
+    use super::*;
+
+    fn partition_metadata(batch: &RecordBatch) -> CheckpointPartitionMetadata {
+        let values = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::()
+            .expect("metadata column");
+        serde_json::from_str(values.value(0)).expect("valid checkpoint metadata")
+    }
+
+    #[test]
+    fn storage_schema_uses_positional_names_and_preserves_metadata() {
+        let logical = Arc::new(Schema::new_with_metadata(
+            vec![
+                Field::new("value", DataType::Int64, false).with_metadata(
+                    [("field-key".to_string(), "field-value".to_string())]
+                        .into_iter()
+                        .collect(),
+                ),
+                Field::new("value", DataType::Utf8, true),
+            ],
+            [("schema-key".to_string(), "schema-value".to_string())]
+                .into_iter()
+                .collect(),
+        ));
+
+        let storage = checkpoint_storage_schema(&logical);
+
+        assert_eq!(storage.field(0).name(), "_c0");
+        assert_eq!(storage.field(1).name(), "_c1");
+        assert_eq!(storage.field(0).metadata(), logical.field(0).metadata());
+        assert_eq!(storage.metadata(), logical.metadata());
+
+        let empty = checkpoint_storage_schema(&Arc::new(Schema::empty()));
+        assert!(empty.fields().is_empty());
+    }
+
+    #[tokio::test]
+    async fn writes_a_partition_directly_to_object_storage() -> Result<()> {
+        let logical_schema = Arc::new(Schema::new(vec![Field::new(
+            "value",
+            DataType::Int32,
+            false,
+        )]));
+        let storage_schema = Arc::new(Schema::new(vec![Field::new("_c0", DataType::Int32, false)]));
+        let batch = RecordBatch::try_new(
+            Arc::clone(&logical_schema),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )?;
+        let input =
+            MemorySourceConfig::try_new_exec(&[vec![batch]], Arc::clone(&logical_schema), None)?;
+        let object_store_url = ObjectStoreUrl::parse("memory://")?;
+        let checkpoint = RemoteCheckpointWriteExec::try_new(
+            input,
+            object_store_url.clone(),
+            Path::from("checkpoint"),
+            storage_schema,
+        )?;
+        let context = SessionContext::new();
+        context
+            .runtime_env()
+            .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new()));
+
+        let mut output = checkpoint.execute(0, context.task_ctx())?;
+        let commit = output.next().await.expect("commit batch")?;
+        let metadata = partition_metadata(&commit);
+        assert_eq!(metadata.partition, 0);
+        assert_eq!(metadata.row_count, 3);
+        let path = Path::parse(metadata.location.expect("checkpoint file"))
+            .map_err(|error| internal_datafusion_err!("invalid path: {error}"))?;
+        assert!(
+            path.filename()
+                .is_some_and(|name| name.starts_with("part-00000-"))
+        );
+        let store = context.runtime_env().object_store(&object_store_url)?;
+        let bytes = store
+            .get(&path)
+            .await
+            .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?
+            .bytes()
+            .await
+            .map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?;
+        let rows = ParquetRecordBatchReaderBuilder::try_new(bytes)?
+            .build()?
+            .map(|batch| batch.map(|batch| batch.num_rows()))
+            .collect::, _>>()?;
+        assert_eq!(rows, vec![3]);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn empty_partition_does_not_create_an_object() -> Result<()> {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "value",
+            DataType::Int32,
+            false,
+        )]));
+        let input = MemorySourceConfig::try_new_exec(&[vec![]], Arc::clone(&schema), None)?;
+        let object_store_url = ObjectStoreUrl::parse("memory://")?;
+        let checkpoint = RemoteCheckpointWriteExec::try_new(
+            input,
+            object_store_url.clone(),
+            Path::from("checkpoint"),
+            schema,
+        )?;
+        let context = SessionContext::new();
+        context
+            .runtime_env()
+            .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new()));
+
+        let mut output = checkpoint.execute(0, context.task_ctx())?;
+        let commit = output.next().await.expect("commit batch")?;
+        let metadata = partition_metadata(&commit);
+        assert!(metadata.location.is_none());
+        assert_eq!(metadata.row_count, 0);
+        assert_eq!(metadata.size, 0);
+        let store = context.runtime_env().object_store(&object_store_url)?;
+        assert!(
+            store
+                .list(Some(&Path::from("checkpoint")))
+                .next()
+                .await
+                .is_none()
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn zero_column_partition_records_rows_without_an_object() -> Result<()> {
+        let schema = Arc::new(Schema::empty());
+        let batch = RecordBatch::try_new_with_options(
+            Arc::clone(&schema),
+            vec![],
+            &RecordBatchOptions::new().with_row_count(Some(3)),
+        )?;
+        let input = MemorySourceConfig::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+        let registry = Arc::new(RemoteCheckpointRegistry::new(
+            Some("memory:///checkpoint".to_string()),
+            "session".to_string(),
+        ));
+        let config = SessionConfig::new().with_extension(Arc::clone(®istry));
+        let context = SessionContext::new_with_config(config);
+        let object_store_url = ObjectStoreUrl::parse("memory://")?;
+        context
+            .runtime_env()
+            .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new()));
+        let (object_store_url, prefix) =
+            registry.resolve_relation(context.runtime_env().as_ref(), "zero-columns")?;
+        let writer: Arc = Arc::new(RemoteCheckpointWriteExec::try_new(
+            input,
+            object_store_url.clone(),
+            prefix.clone(),
+            Arc::clone(&schema),
+        )?);
+        let checkpoint = RemoteCheckpointCommitExec::new(
+            Arc::new(CoalescePartitionsExec::new(writer)),
+            "zero-columns".to_string(),
+            object_store_url.clone(),
+            prefix.clone(),
+            Arc::clone(&schema),
+            schema,
+            Partitioning::UnknownPartitioning(1),
+            None,
+        );
+
+        let mut output = checkpoint.execute(0, context.task_ctx())?;
+        output.next().await.expect("checkpoint commit batch")?;
+        let descriptor = registry
+            .get("zero-columns")?
+            .expect("checkpoint descriptor must be published");
+        assert_eq!(descriptor.partitions[0].row_count, 3);
+        assert!(descriptor.partitions[0].file.is_none());
+        let store = context.runtime_env().object_store(&object_store_url)?;
+        assert!(store.list(Some(&prefix)).next().await.is_none());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn commit_publishes_checkpoint_and_registry_cleans_it() -> Result<()> {
+        let logical_schema = Arc::new(Schema::new(vec![Field::new(
+            "value",
+            DataType::Int32,
+            false,
+        )]));
+        let storage_schema = checkpoint_storage_schema(&logical_schema);
+        let batch = RecordBatch::try_new(
+            Arc::clone(&logical_schema),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )?;
+        let input = MemorySourceConfig::try_new_exec(
+            &[vec![batch], vec![]],
+            Arc::clone(&logical_schema),
+            None,
+        )?;
+        let registry = Arc::new(RemoteCheckpointRegistry::new(
+            Some("memory:///checkpoint".to_string()),
+            "session".to_string(),
+        ));
+        let config = SessionConfig::new().with_extension(Arc::clone(®istry));
+        let context = SessionContext::new_with_config(config);
+        let object_store_url = ObjectStoreUrl::parse("memory://")?;
+        context
+            .runtime_env()
+            .register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new()));
+        let (object_store_url, prefix) =
+            registry.resolve_relation(context.runtime_env().as_ref(), "relation")?;
+        let writer: Arc = Arc::new(RemoteCheckpointWriteExec::try_new(
+            input,
+            object_store_url.clone(),
+            prefix.clone(),
+            Arc::clone(&storage_schema),
+        )?);
+        let commit = RemoteCheckpointCommitExec::new(
+            Arc::new(CoalescePartitionsExec::new(writer)),
+            "relation".to_string(),
+            object_store_url.clone(),
+            prefix.clone(),
+            Arc::clone(&logical_schema),
+            storage_schema,
+            Partitioning::UnknownPartitioning(1),
+            None,
+        );
+
+        let mut output = commit.execute(0, context.task_ctx())?;
+        output.next().await.expect("checkpoint commit batch")?;
+        let descriptor = registry
+            .get("relation")?
+            .expect("checkpoint descriptor must be published");
+        assert!(matches!(
+            descriptor.output_partitioning,
+            Partitioning::UnknownPartitioning(2)
+        ));
+        assert_eq!(descriptor.partitions.len(), 2);
+        assert_eq!(descriptor.partitions[0].row_count, 3);
+        assert_eq!(descriptor.partitions[1].row_count, 0);
+        assert!(descriptor.partitions[0].file.is_some());
+        assert!(descriptor.partitions[1].file.is_none());
+        let store = context.runtime_env().object_store(&object_store_url)?;
+        assert!(store.list(Some(&prefix)).next().await.is_some());
+
+        registry
+            .cleanup_session(context.runtime_env().as_ref())
+            .await?;
+        assert!(registry.get("relation")?.is_none());
+        assert!(store.list(Some(&prefix)).next().await.is_none());
+        assert!(
+            registry
+                .resolve_relation(context.runtime_env().as_ref(), "another")
+                .is_err()
+        );
+        Ok(())
+    }
+}
diff --git a/crates/sail-physical-plan/src/repartition.rs b/crates/sail-physical-plan/src/repartition.rs
index e47339a9e..fb98c0411 100644
--- a/crates/sail-physical-plan/src/repartition.rs
+++ b/crates/sail-physical-plan/src/repartition.rs
@@ -12,17 +12,20 @@ use datafusion::physical_expr::Partitioning;
 use datafusion::physical_plan::execution_plan::{
     CardinalityEffect, EvaluationType, SchedulingType,
 };
+use datafusion::physical_plan::projection::{
+    ProjectionExec, all_columns, make_with_child, update_expr,
+};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use datafusion_common::{internal_err, plan_err, Result, Statistics};
+use datafusion_common::{Result, Statistics, internal_err, plan_err};
 use futures::{Stream, StreamExt};
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::repartition::{
-    RepartitionBufferConfig, DEFAULT_REPARTITION_BUFFER_SIZE,
+    DEFAULT_REPARTITION_BUFFER_SIZE, RepartitionBufferConfig,
 };
-use tokio::sync::mpsc::{channel, Receiver, Sender};
+use tokio::sync::mpsc::{Receiver, Sender, channel};
 
 #[derive(Debug, Clone)]
 pub struct RowRoundRobinPartitioner {
@@ -483,10 +486,43 @@ impl ExecutionPlan for ExplicitRepartitionExec {
         CardinalityEffect::Equal
     }
 
-    // TODO: Implement the logic to push down filters or projections.
-    //   The filters and projections are safe to push down if they are
-    //   column references. For other expressions, we may not want to
-    //   push them down since the evaluation can be potentially expensive,
-    //   and the presence of explicit repartitioning indicates that the user
-    //   wants to evaluate these expressions after repartitioning.
+    fn try_swapping_with_projection(
+        &self,
+        projection: &ProjectionExec,
+    ) -> Result>> {
+        // No benefit if projection does not narrow the schema
+        if projection.expr().len() >= projection.input().schema().fields().len() {
+            return Ok(None);
+        }
+
+        // Skip if projection expressions are not simple column references
+        if projection.benefits_from_input_partitioning()[0] || !all_columns(projection.expr()) {
+            return Ok(None);
+        }
+
+        let new_projection = make_with_child(projection, self.input())?;
+        // Handle all partitioning variants — 'ProjectionPushdown' runs before 'RewriteExplicitRepartition'
+        let new_partitioning = match &self.properties.partitioning {
+            Partitioning::Hash(partitions, size) => {
+                let mut new_partitions = vec![];
+                for partition in partitions {
+                    let Some(new_partition) = update_expr(partition, projection.expr(), false)?
+                    else {
+                        return Ok(None);
+                    };
+                    new_partitions.push(new_partition);
+                }
+                Partitioning::Hash(new_partitions, *size)
+            }
+            other => other.clone(),
+        };
+        Ok(Some(Arc::new(ExplicitRepartitionExec::new(
+            new_projection,
+            new_partitioning,
+        ))))
+    }
+
+    // Do not implement Filter pushdown, as its evaluation can be potentially
+    // expensive, and the presence of explicit repartitioning indicates that
+    // the user wants to evaluate the filters after repartitioning.
 }
diff --git a/crates/sail-physical-plan/src/schema_pivot.rs b/crates/sail-physical-plan/src/schema_pivot.rs
index b63af9575..2e91b950b 100644
--- a/crates/sail-physical-plan/src/schema_pivot.rs
+++ b/crates/sail-physical-plan/src/schema_pivot.rs
@@ -10,7 +10,7 @@ use datafusion::physical_expr::{EquivalenceProperties, Partitioning};
 use datafusion::physical_plan::{
     DisplayAs, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use datafusion_common::{exec_err, internal_datafusion_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, exec_err, internal_datafusion_err};
 use futures::{Stream, StreamExt};
 use sail_common_datafusion::utils::items::ItemTaker;
 
@@ -200,7 +200,7 @@ impl Stream for SchemaPivotStream {
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> {
         loop {
-            match self.poll(cx) {
+            match self.as_mut().get_mut().poll(cx) {
                 Poll::Pending => break Poll::Pending,
                 Poll::Ready(SchemaPivotState::SchemaPivot) => {
                     self.input = None;
diff --git a/crates/sail-physical-plan/src/show_string.rs b/crates/sail-physical-plan/src/show_string.rs
index c3676de1b..b9efd51fd 100644
--- a/crates/sail-physical-plan/src/show_string.rs
+++ b/crates/sail-physical-plan/src/show_string.rs
@@ -11,7 +11,7 @@ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
 use datafusion::physical_plan::{
     DisplayAs, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use datafusion_common::{exec_err, internal_datafusion_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, exec_err, internal_datafusion_err};
 use futures::{Stream, StreamExt};
 use sail_common_datafusion::rename::physical_plan::rename_physical_plan;
 use sail_common_datafusion::utils::items::ItemTaker;
@@ -217,7 +217,7 @@ impl Stream for ShowStringStream {
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> {
         loop {
-            match self.poll(cx) {
+            match self.as_mut().get_mut().poll(cx) {
                 Poll::Pending => break Poll::Pending,
                 Poll::Ready(ShowStringState::Show) => {
                     self.input = None;
diff --git a/crates/sail-physical-plan/src/spark_partition_id.rs b/crates/sail-physical-plan/src/spark_partition_id.rs
index d044044e8..3f25ce05e 100644
--- a/crates/sail-physical-plan/src/spark_partition_id.rs
+++ b/crates/sail-physical-plan/src/spark_partition_id.rs
@@ -12,7 +12,7 @@ use datafusion::physical_plan::{
     RecordBatchStream,
 };
 use datafusion_common::stats::Precision;
-use datafusion_common::{exec_err, internal_err, ColumnStatistics, Result, Statistics};
+use datafusion_common::{ColumnStatistics, Result, Statistics, exec_err, internal_err};
 use futures::Stream;
 
 #[derive(Debug, Clone)]
@@ -42,7 +42,8 @@ impl SparkPartitionIdExec {
         }
 
         let properties = Arc::new(PlanProperties::new(
-            EquivalenceProperties::new(schema.clone()),
+            EquivalenceProperties::new(schema.clone())
+                .extend(input.equivalence_properties().clone())?,
             input.output_partitioning().clone(),
             input.pipeline_behavior(),
             input.boundedness(),
diff --git a/crates/sail-physical-plan/src/streaming/collector.rs b/crates/sail-physical-plan/src/streaming/collector.rs
index a59d452c6..2ed9acc2e 100644
--- a/crates/sail-physical-plan/src/streaming/collector.rs
+++ b/crates/sail-physical-plan/src/streaming/collector.rs
@@ -7,11 +7,11 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
     DisplayAs, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use datafusion_common::{internal_err, plan_err, Result};
+use datafusion_common::{Result, internal_err, plan_err};
 use futures::StreamExt;
+use sail_common_datafusion::streaming::event::FlowEvent;
 use sail_common_datafusion::streaming::event::encoding::DecodedFlowEventStream;
 use sail_common_datafusion::streaming::event::schema::try_from_flow_event_schema;
-use sail_common_datafusion::streaming::event::FlowEvent;
 
 /// A physical plan node that collects a stream of retractable data batches
 /// into final data batches.
diff --git a/crates/sail-physical-plan/src/streaming/limit.rs b/crates/sail-physical-plan/src/streaming/limit.rs
index 8e29dec98..49c9d0980 100644
--- a/crates/sail-physical-plan/src/streaming/limit.rs
+++ b/crates/sail-physical-plan/src/streaming/limit.rs
@@ -8,8 +8,9 @@ use datafusion::physical_plan::{
     DisplayAs, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
 use datafusion_common::arrow::datatypes::SchemaRef;
-use datafusion_common::{internal_err, plan_err, Result, Statistics};
+use datafusion_common::{Result, Statistics, internal_err, plan_err};
 use futures::StreamExt;
+use sail_common_datafusion::streaming::event::FlowEvent;
 use sail_common_datafusion::streaming::event::encoding::{
     DecodedFlowEventStream, EncodedFlowEventStream,
 };
@@ -17,7 +18,6 @@ use sail_common_datafusion::streaming::event::schema::try_from_flow_event_schema
 use sail_common_datafusion::streaming::event::stream::{
     FlowEventStreamAdapter, SendableFlowEventStream,
 };
-use sail_common_datafusion::streaming::event::FlowEvent;
 
 /// A physical plan node that limits the number of retractable rows during
 /// streaming query execution.
@@ -221,17 +221,21 @@ impl StreamLimitState {
         match event {
             Some(m @ Ok(FlowEvent::Marker(_))) => Some((Some(m), this)),
             Some(Ok(FlowEvent::Data { batch, retracted })) => {
-                if let Some((batch, retracted)) = this.skip_batch_if_needed(batch, retracted) {
-                    if let Some((batch, retracted)) = this.fetch_batch(batch, retracted) {
-                        Some((Some(Ok(FlowEvent::Data { batch, retracted })), this))
-                    } else {
-                        // TODO: Here we close the stream when fetch limit is reached.
-                        //   But we need to send the end-of-data marker and propagate
-                        //   future markers until the next checkpoint has completed.
-                        None
+                match this.skip_batch_if_needed(batch, retracted) {
+                    Some((batch, retracted)) => {
+                        match this.fetch_batch(batch, retracted) {
+                            Some((batch, retracted)) => {
+                                Some((Some(Ok(FlowEvent::Data { batch, retracted })), this))
+                            }
+                            _ => {
+                                // TODO: Here we close the stream when fetch limit is reached.
+                                //   But we need to send the end-of-data marker and propagate
+                                //   future markers until the next checkpoint has completed.
+                                None
+                            }
+                        }
                     }
-                } else {
-                    Some((None, this))
+                    _ => Some((None, this)),
                 }
             }
             Some(Err(e)) => Some((Some(Err(e)), this)),
diff --git a/crates/sail-physical-plan/src/streaming/source_adapter.rs b/crates/sail-physical-plan/src/streaming/source_adapter.rs
index 448e5e28a..24bb43b39 100644
--- a/crates/sail-physical-plan/src/streaming/source_adapter.rs
+++ b/crates/sail-physical-plan/src/streaming/source_adapter.rs
@@ -5,13 +5,13 @@ use datafusion::physical_expr::EquivalenceProperties;
 use datafusion::physical_plan::{
     DisplayAs, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use datafusion_common::{plan_err, Result};
-use futures::{stream, StreamExt};
+use datafusion_common::{Result, plan_err};
+use futures::{StreamExt, stream};
+use sail_common_datafusion::streaming::event::FlowEvent;
 use sail_common_datafusion::streaming::event::encoding::EncodedFlowEventStream;
 use sail_common_datafusion::streaming::event::marker::FlowMarker;
 use sail_common_datafusion::streaming::event::schema::to_flow_event_schema;
 use sail_common_datafusion::streaming::event::stream::FlowEventStreamAdapter;
-use sail_common_datafusion::streaming::event::FlowEvent;
 
 /// A physical plan node that adapts a non-streaming source to a streaming source.
 /// The input schema is the original data schema, while the output schema is the
diff --git a/crates/sail-plan/Cargo.toml b/crates/sail-plan/Cargo.toml
index 6962c046e..f3ed96e15 100644
--- a/crates/sail-plan/Cargo.toml
+++ b/crates/sail-plan/Cargo.toml
@@ -17,6 +17,7 @@ sail-function = { path = "../sail-function" }
 sail-logical-plan = { path = "../sail-logical-plan" }
 sail-physical-plan = { path = "../sail-physical-plan" }
 sail-data-source = { path = "../sail-data-source" }
+sail-cache = { path = "../sail-cache" }
 
 datafusion = { workspace = true }
 datafusion-common = { workspace = true }
@@ -47,4 +48,9 @@ serde_json = { workspace = true }
 indexmap = { workspace = true }
 log = { workspace = true }
 regex = { workspace = true }
+regex-syntax = { workspace = true }
 parquet-variant-compute = { workspace = true }
+
+[build-dependencies]
+serde = { workspace = true }
+serde_yaml = { workspace = true }
diff --git a/crates/sail-plan/build.rs b/crates/sail-plan/build.rs
new file mode 100644
index 000000000..0fe743427
--- /dev/null
+++ b/crates/sail-plan/build.rs
@@ -0,0 +1,238 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use serde::Deserialize;
+
+// TODO: Make metadata minimal and structured with stricter enums and defaults.
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct FunctionMetadataEntry {
+    name: String,
+    #[serde(default)]
+    scope: Option,
+    #[serde(default)]
+    group: Option,
+    #[serde(default)]
+    source: Option,
+    #[serde(default)]
+    usage: Option,
+    #[serde(default)]
+    arguments: Option,
+    #[serde(default)]
+    examples: Option,
+    #[serde(default)]
+    note: Option,
+    #[serde(default)]
+    since: Option,
+    #[serde(default)]
+    deprecated: Option,
+    #[serde(default)]
+    class_name: Option,
+    #[serde(default)]
+    signatures: Vec,
+}
+
+impl FunctionMetadataEntry {
+    fn validate(&self) -> Result<(), Box> {
+        if self.name.is_empty() {
+            return Err("function metadata entry has an empty name".into());
+        }
+        if self.name != self.name.to_ascii_lowercase() {
+            return Err(format!("function metadata name must be lowercase: {}", self.name).into());
+        }
+        let Some(scope) = self.scope.as_deref() else {
+            return Err(format!("function metadata is missing scope: {}", self.name).into());
+        };
+        if !matches!(
+            scope,
+            "sql_builtin" | "spark_internal" | "pyspark_only" | "sail_extension"
+        ) {
+            return Err(format!(
+                "function metadata has an unknown scope for {}: {}",
+                self.name, scope
+            )
+            .into());
+        }
+        if self.group.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has an empty group: {}", self.name).into());
+        }
+        if self.source.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has an empty source: {}", self.name).into());
+        }
+        if self.usage.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has empty usage: {}", self.name).into());
+        }
+        if self.arguments.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has empty arguments: {}", self.name).into());
+        }
+        if self.examples.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has empty examples: {}", self.name).into());
+        }
+        if self.note.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has an empty note: {}", self.name).into());
+        }
+        if self.since.as_deref().is_some_and(str::is_empty) {
+            return Err(
+                format!("function metadata has an empty since value: {}", self.name).into(),
+            );
+        }
+        if self.deprecated.as_deref().is_some_and(str::is_empty) {
+            return Err(format!("function metadata has empty deprecated: {}", self.name).into());
+        }
+        if self.signatures.iter().any(|signature| signature.is_empty()) {
+            return Err(format!("function metadata has an empty signature: {}", self.name).into());
+        }
+        if self.usage.as_deref().is_some_and(|usage| !usage.is_empty())
+            && self.signatures.is_empty()
+        {
+            return Err(format!(
+                "function metadata has usage but no signature: {}",
+                self.name
+            )
+            .into());
+        }
+        if scope == "sql_builtin" {
+            if self.group.is_none() {
+                return Err(
+                    format!("built-in function metadata is missing group: {}", self.name).into(),
+                );
+            }
+            if self.source.is_none() {
+                return Err(format!(
+                    "built-in function metadata is missing source: {}",
+                    self.name
+                )
+                .into());
+            }
+        }
+        Ok(())
+    }
+}
+
+fn rust_string(value: &str) -> String {
+    format!("{value:?}")
+}
+
+fn rust_option_string(value: Option<&str>) -> String {
+    match value {
+        Some(value) => format!("Some({})", rust_string(value)),
+        None => "None".to_string(),
+    }
+}
+
+fn load_function_metadata_file(
+    path: &Path,
+) -> Result, Box> {
+    println!("cargo:rerun-if-changed={}", path.display());
+    let content = std::fs::read_to_string(path)?;
+    let entries: Vec = serde_yaml::from_str(&content)?;
+
+    let mut previous = None;
+    for entry in &entries {
+        entry.validate()?;
+        if previous.is_some_and(|previous| previous > entry.name.as_str()) {
+            return Err(format!(
+                "function metadata entries must be sorted by name: {}",
+                path.display()
+            )
+            .into());
+        }
+        previous = Some(entry.name.as_str());
+    }
+    Ok(entries)
+}
+
+fn collect_function_metadata_files(
+    path: &Path,
+    output: &mut Vec,
+) -> Result<(), Box> {
+    println!("cargo:rerun-if-changed={}", path.display());
+    let mut entries = std::fs::read_dir(path)?.collect::, _>>()?;
+    entries.sort_by_key(|entry| entry.path());
+    for entry in entries {
+        let path = entry.path();
+        let file_type = entry.file_type()?;
+        if file_type.is_dir() {
+            collect_function_metadata_files(&path, output)?;
+        } else if file_type.is_file()
+            && path
+                .extension()
+                .and_then(|extension| extension.to_str())
+                .is_some_and(|extension| matches!(extension, "yaml" | "yml"))
+        {
+            output.push(path);
+        }
+    }
+    Ok(())
+}
+
+fn load_function_metadata(
+    path: &Path,
+) -> Result, Box> {
+    let mut paths = Vec::new();
+    collect_function_metadata_files(path, &mut paths)?;
+    if paths.is_empty() {
+        return Err(format!("no function metadata files found in {}", path.display()).into());
+    }
+
+    let mut entries = Vec::new();
+    let mut seen = HashMap::new();
+    for path in paths {
+        for entry in load_function_metadata_file(&path)? {
+            if let Some(previous_path) = seen.insert(entry.name.clone(), path.display().to_string())
+            {
+                return Err(format!(
+                    "duplicate function metadata entry: {} in {} and {}",
+                    entry.name,
+                    previous_path,
+                    path.display()
+                )
+                .into());
+            }
+            entries.push(entry);
+        }
+    }
+    entries.sort_by(|left, right| left.name.cmp(&right.name));
+    Ok(entries)
+}
+
+fn build_function_metadata(
+    entries: &[FunctionMetadataEntry],
+    out_dir: &Path,
+) -> Result<(), Box> {
+    // TODO: Generate Rust code with `quote` instead of formatting strings.
+    let mut output = String::new();
+    output
+        .push_str("pub(crate) const BUILT_IN_FUNCTION_METADATA: &[BuiltInFunctionMetadata] = &[\n");
+    for entry in entries {
+        let name = rust_string(&entry.name);
+        let is_public = entry.scope.as_deref() == Some("sql_builtin");
+        let signatures = entry
+            .signatures
+            .iter()
+            .map(|signature| rust_string(signature))
+            .collect::>()
+            .join(", ");
+        let usage = rust_option_string(entry.usage.as_deref());
+        let arguments = rust_option_string(entry.arguments.as_deref());
+        let examples = rust_option_string(entry.examples.as_deref());
+        let note = rust_option_string(entry.note.as_deref());
+        let since = rust_option_string(entry.since.as_deref());
+        let deprecated = rust_option_string(entry.deprecated.as_deref());
+        let class_name = rust_string(entry.class_name.as_deref().unwrap_or(""));
+        output.push_str(&format!(
+            "    BuiltInFunctionMetadata {{ name: {name}, signatures: &[{signatures}], usage: {usage}, arguments: {arguments}, examples: {examples}, note: {note}, since: {since}, deprecated: {deprecated}, class_name: {class_name}, is_public: {is_public} }},\n"
+        ));
+    }
+    output.push_str("];\n");
+    std::fs::write(out_dir.join("function_metadata.rs"), output)?;
+    Ok(())
+}
+
+fn main() -> Result<(), Box> {
+    let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?);
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR")?);
+    let entries = load_function_metadata(&manifest_dir.join("data/functions"))?;
+    build_function_metadata(&entries, &out_dir)?;
+    Ok(())
+}
diff --git a/crates/sail-plan/data/functions/aggregate.yaml b/crates/sail-plan/data/functions/aggregate.yaml
new file mode 100644
index 000000000..aae11ada6
--- /dev/null
+++ b/crates/sail-plan/data/functions/aggregate.yaml
@@ -0,0 +1,1841 @@
+- name: "any"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr"
+  signatures:
+    - "any(expr)"
+  usage: "any(expr) - Returns true if at least one value of `expr` is true."
+  examples: |2-
+    
+        Examples:
+          > SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col);
+           true
+          > SELECT any(col) FROM VALUES (NULL), (true), (false) AS tab(col);
+           true
+          > SELECT any(col) FROM VALUES (false), (false), (NULL) AS tab(col);
+           false
+      
+  since: "3.0.0"
+
+- name: "any_value"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.AnyValue"
+  signatures:
+    - "any_value(expr[, isIgnoreNull])"
+  usage: |2-
+    
+        any_value(expr[, isIgnoreNull]) - Returns some value of `expr` for a group of rows.
+          If `isIgnoreNull` is true, returns only non-null values.
+  examples: |2-
+    
+        Examples:
+          > SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col);
+           10
+          > SELECT any_value(col) FROM VALUES (NULL), (5), (20) AS tab(col);
+           NULL
+          > SELECT any_value(col, true) FROM VALUES (NULL), (5), (20) AS tab(col);
+           5
+      
+  note: |2-
+    
+        The function is non-deterministic.
+      
+  since: "3.4.0"
+
+- name: "approx_count_distinct"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus"
+  signatures:
+    - "approx_count_distinct(expr[, relativeSD])"
+  usage: |2-
+    
+        approx_count_distinct(expr[, relativeSD]) - Returns the estimated cardinality by HyperLogLog++.
+          `relativeSD` defines the maximum relative standard deviation allowed.
+  examples: |2-
+    
+        Examples:
+          > SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);
+           3
+      
+  since: "1.6.0"
+
+- name: "approx_percentile"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile"
+  signatures:
+    - "approx_percentile(col, percentage [, accuracy])"
+  usage: |2-
+    
+        approx_percentile(col, percentage [, accuracy]) - Returns the approximate `percentile` of the numeric or
+          ansi interval column `col` which is the smallest value in the ordered `col` values (sorted
+          from least to greatest) such that no more than `percentage` of `col` values is less than
+          the value or equal to that value. The value of percentage must be between 0.0 and 1.0.
+          The `accuracy` parameter (default: 10000) is a positive numeric literal which controls
+          approximation accuracy at the cost of memory. Higher value of `accuracy` yields better
+          accuracy, `1.0/accuracy` is the relative error of the approximation.
+          When `percentage` is an array, each value of the percentage array must be between 0.0 and 1.0.
+          In this case, returns the approximate percentile array of column `col` at the given
+          percentage array.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);
+           [1,1,0]
+          > SELECT approx_percentile(col, 0.5, 100) FROM VALUES (0), (6), (7), (9), (10) AS tab(col);
+           7
+          > SELECT approx_percentile(col, 0.5, 100) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '1' MONTH), (INTERVAL '2' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-1
+          > SELECT approx_percentile(col, array(0.5, 0.7), 100) FROM VALUES (INTERVAL '0' SECOND), (INTERVAL '1' SECOND), (INTERVAL '2' SECOND), (INTERVAL '10' SECOND) AS tab(col);
+           [0 00:00:01.000000000,0 00:00:02.000000000]
+      
+  since: "2.1.0"
+
+- name: "array_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CollectList"
+  signatures:
+    - "array_agg(expr)"
+  usage: "array_agg(expr) - Collects and returns a list of non-unique elements."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col);
+           [1,2,1]
+      
+  note: |2-
+    
+        The function is non-deterministic because the order of collected results depends
+        on the order of the rows which may be non-deterministic after a shuffle.
+      
+  since: "3.3.0"
+
+- name: "avg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Average"
+  signatures:
+    - "avg(expr)"
+  usage: "avg(expr) - Returns the mean calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col);
+           2.0
+          > SELECT avg(col) FROM VALUES (1), (2), (NULL) AS tab(col);
+           1.5
+      
+  since: "1.0.0"
+
+- name: "bit_and"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BitAndAgg"
+  signatures:
+    - "bit_and(expr)"
+  usage: "bit_and(expr) - Returns the bitwise AND of all non-null input values, or null if none."
+  examples: |2-
+    
+        Examples:
+          > SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col);
+           1
+      
+  since: "3.0.0"
+
+- name: "bit_or"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BitOrAgg"
+  signatures:
+    - "bit_or(expr)"
+  usage: "bit_or(expr) - Returns the bitwise OR of all non-null input values, or null if none."
+  examples: |2-
+    
+        Examples:
+          > SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col);
+           7
+      
+  since: "3.0.0"
+
+- name: "bit_xor"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BitXorAgg"
+  signatures:
+    - "bit_xor(expr)"
+  usage: "bit_xor(expr) - Returns the bitwise XOR of all non-null input values, or null if none."
+  examples: |2-
+    
+        Examples:
+          > SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col);
+           6
+      
+  since: "3.0.0"
+
+- name: "bitmap_and_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitmapAndAgg"
+  signatures:
+    - "bitmap_and_agg(child)"
+  usage: |2-
+    
+        bitmap_and_agg(child) - Returns a bitmap that is the bitwise AND of all of the bitmaps from the child
+        expression. The input should be bitmaps created from bitmap_construct_agg().
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT substring(hex(bitmap_and_agg(col)), 0, 6) FROM VALUES (X 'F0'), (X '70'), (X '30') AS tab(col);
+           300000
+          > SELECT substring(hex(bitmap_and_agg(col)), 0, 6) FROM VALUES (X 'FF'), (X 'FF'), (X 'FF') AS tab(col);
+           FF0000
+      
+  since: "4.1.0"
+
+- name: "bitmap_construct_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitmapConstructAgg"
+  signatures:
+    - "bitmap_construct_agg(child)"
+  usage: |2-
+    
+        bitmap_construct_agg(child) - Returns a bitmap with the positions of the bits set from all the values from
+        the child expression. The child expression will most likely be bitmap_bit_position().
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) FROM VALUES (1), (2), (3) AS tab(col);
+           070000
+          > SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) FROM VALUES (1), (1), (1) AS tab(col);
+           010000
+      
+  since: "3.5.0"
+
+- name: "bitmap_or_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitmapOrAgg"
+  signatures:
+    - "bitmap_or_agg(child)"
+  usage: |2-
+    
+        bitmap_or_agg(child) - Returns a bitmap that is the bitwise OR of all of the bitmaps from the child
+        expression. The input should be bitmaps created from bitmap_construct_agg().
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT substring(hex(bitmap_or_agg(col)), 0, 6) FROM VALUES (X '10'), (X '20'), (X '40') AS tab(col);
+           700000
+          > SELECT substring(hex(bitmap_or_agg(col)), 0, 6) FROM VALUES (X '10'), (X '10'), (X '10') AS tab(col);
+           100000
+      
+  since: "3.5.0"
+
+- name: "bool_and"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd"
+  signatures:
+    - "bool_and(expr)"
+  usage: "bool_and(expr) - Returns true if all values of `expr` are true."
+  examples: |2-
+    
+        Examples:
+          > SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col);
+           true
+          > SELECT bool_and(col) FROM VALUES (NULL), (true), (true) AS tab(col);
+           true
+          > SELECT bool_and(col) FROM VALUES (true), (false), (true) AS tab(col);
+           false
+      
+  since: "3.0.0"
+
+- name: "bool_or"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr"
+  signatures:
+    - "bool_or(expr)"
+  usage: "bool_or(expr) - Returns true if at least one value of `expr` is true."
+  examples: |2-
+    
+        Examples:
+          > SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col);
+           true
+          > SELECT bool_or(col) FROM VALUES (NULL), (true), (false) AS tab(col);
+           true
+          > SELECT bool_or(col) FROM VALUES (false), (false), (NULL) AS tab(col);
+           false
+      
+  since: "3.0.0"
+
+- name: "collect_list"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CollectList"
+  signatures:
+    - "collect_list(expr)"
+  usage: "collect_list(expr) - Collects and returns a list of non-unique elements."
+  examples: |2-
+    
+        Examples:
+          > SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col);
+           [1,2,1]
+      
+  note: |2-
+    
+        The function is non-deterministic because the order of collected results depends
+        on the order of the rows which may be non-deterministic after a shuffle.
+      
+  since: "2.0.0"
+
+- name: "collect_set"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet"
+  signatures:
+    - "collect_set(expr)"
+  usage: "collect_set(expr) - Collects and returns a set of unique elements."
+  examples: |2-
+    
+        Examples:
+          > SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col);
+           [1,2]
+      
+  note: |2-
+    
+        The function is non-deterministic because the order of collected results depends
+        on the order of the rows which may be non-deterministic after a shuffle.
+      
+  since: "2.0.0"
+
+- name: "corr"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Corr"
+  signatures:
+    - "corr(expr1, expr2)"
+  usage: "corr(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number pairs."
+  examples: |2-
+    
+        Examples:
+          > SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2);
+           0.8660254037844387
+      
+  since: "1.6.0"
+
+- name: "count"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Count"
+  signatures:
+    - "count(*)"
+    - "count(expr[, expr...])"
+    - "count(DISTINCT expr[, expr...])"
+  usage: |2-
+    
+        count(*) - Returns the total number of retrieved rows, including rows containing null.
+    
+        count(expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are all non-null.
+    
+        count(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-null.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col);
+           4
+          > SELECT count(col) FROM VALUES (NULL), (5), (5), (20) AS tab(col);
+           3
+          > SELECT count(DISTINCT col) FROM VALUES (NULL), (5), (5), (10) AS tab(col);
+           2
+      
+  since: "1.0.0"
+
+- name: "count_if"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CountIf"
+  signatures:
+    - "count_if(expr)"
+  usage: |2-
+    
+        count_if(expr) - Returns the number of `TRUE` values for the expression.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col);
+           2
+          > SELECT count_if(col IS NULL) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col);
+           1
+      
+  since: "3.0.0"
+
+- name: "count_min_sketch"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CountMinSketchAggExpressionBuilder"
+  signatures:
+    - "count_min_sketch(col, eps, confidence, seed)"
+  usage: |2-
+    
+        count_min_sketch(col, eps, confidence, seed) - Returns a count-min sketch of a column with the given esp,
+          confidence and seed. The result is an array of bytes, which can be deserialized to a
+          `CountMinSketch` before usage. Count-min sketch is a probabilistic data structure used for
+          cardinality estimation using sub-linear space.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col);
+           0000000100000000000000030000000100000004000000005D8D6AB90000000000000000000000000000000200000000000000010000000000000000
+      
+  since: "2.2.0"
+
+- name: "covar_pop"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CovPopulation"
+  signatures:
+    - "covar_pop(expr1, expr2)"
+  usage: "covar_pop(expr1, expr2) - Returns the population covariance of a set of number pairs."
+  examples: |2-
+    
+        Examples:
+          > SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);
+           0.6666666666666666
+      
+  since: "2.0.0"
+
+- name: "covar_samp"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.CovSample"
+  signatures:
+    - "covar_samp(expr1, expr2)"
+  usage: "covar_samp(expr1, expr2) - Returns the sample covariance of a set of number pairs."
+  examples: |2-
+    
+        Examples:
+          > SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);
+           1.0
+      
+  since: "2.0.0"
+
+- name: "every"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd"
+  signatures:
+    - "every(expr)"
+  usage: "every(expr) - Returns true if all values of `expr` are true."
+  examples: |2-
+    
+        Examples:
+          > SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col);
+           true
+          > SELECT every(col) FROM VALUES (NULL), (true), (true) AS tab(col);
+           true
+          > SELECT every(col) FROM VALUES (true), (false), (true) AS tab(col);
+           false
+      
+  since: "3.0.0"
+
+- name: "first"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.First"
+  signatures:
+    - "first(expr[, isIgnoreNull])"
+  usage: |2-
+    
+        first(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows.
+          If `isIgnoreNull` is true, returns only non-null values.
+  examples: |2-
+    
+        Examples:
+          > SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col);
+           10
+          > SELECT first(col) FROM VALUES (NULL), (5), (20) AS tab(col);
+           NULL
+          > SELECT first(col, true) FROM VALUES (NULL), (5), (20) AS tab(col);
+           5
+      
+  note: |2-
+    
+        The function is non-deterministic because its results depends on the order of the rows
+        which may be non-deterministic after a shuffle.
+      
+  since: "2.0.0"
+
+- name: "first_value"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.First"
+  signatures:
+    - "first_value(expr[, isIgnoreNull])"
+  usage: |2-
+    
+        first_value(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows.
+          If `isIgnoreNull` is true, returns only non-null values.
+  examples: |2-
+    
+        Examples:
+          > SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col);
+           10
+          > SELECT first_value(col) FROM VALUES (NULL), (5), (20) AS tab(col);
+           NULL
+          > SELECT first_value(col, true) FROM VALUES (NULL), (5), (20) AS tab(col);
+           5
+      
+  note: |2-
+    
+        The function is non-deterministic because its results depends on the order of the rows
+        which may be non-deterministic after a shuffle.
+      
+  since: "2.0.0"
+
+- name: "grouping"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Grouping"
+  signatures:
+    - "grouping(col)"
+  usage: |2-
+    
+        grouping(col) - indicates whether a specified column in a GROUP BY is aggregated or
+          not, returns 1 for aggregated or 0 for not aggregated in the result set.",
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name);
+            Alice	0	2
+            Bob	0	5
+            NULL	1	7
+      
+  since: "2.0.0"
+
+- name: "grouping_id"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.GroupingID"
+  signatures:
+    - "grouping_id([col1[, col2 ..]])"
+  usage: |2-
+    
+        grouping_id([col1[, col2 ..]]) - returns the level of grouping, equals to
+          `(grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)`
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);
+            Alice	0	2	165.0
+            Alice	1	2	165.0
+            NULL	3	7	172.5
+            Bob	0	5	180.0
+            Bob	1	5	180.0
+            NULL	2	2	165.0
+            NULL	2	5	180.0
+      
+  note: |2-
+    
+        Input columns should match with grouping columns exactly, or empty (means all the grouping
+        columns).
+      
+  since: "2.0.0"
+
+- name: "histogram_numeric"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.HistogramNumeric"
+  signatures:
+    - "histogram_numeric(expr, nb)"
+  usage: |2-
+    
+        histogram_numeric(expr, nb) - Computes a histogram on numeric 'expr' using nb bins.
+          The return value is an array of (x,y) pairs representing the centers of the
+          histogram's bins. As the value of 'nb' is increased, the histogram approximation
+          gets finer-grained, but may yield artifacts around outliers. In practice, 20-40
+          histogram bins appear to work well, with more bins being required for skewed or
+          smaller datasets. Note that this function creates a histogram with non-uniform
+          bin widths. It offers no guarantees in terms of the mean-squared-error of the
+          histogram, but in practice is comparable to the histograms produced by the R/S-Plus
+          statistical computing packages. Note: the output type of the 'x' field in the return value is
+          propagated from the input value consumed in the aggregate function.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col);
+           [{"x":0,"y":1.0},{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":10,"y":1.0}]
+      
+  since: "3.3.0"
+
+- name: "hll_sketch_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.HllSketchAgg"
+  signatures:
+    - "hll_sketch_agg(expr, lgConfigK)"
+  usage: |2-
+    
+        hll_sketch_agg(expr, lgConfigK) - Returns the HllSketch's updatable binary representation.
+          `lgConfigK` (optional) the log-base-2 of K, with K is the number of buckets or
+          slots for the HllSketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT hll_sketch_estimate(hll_sketch_agg(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col);
+           3
+      
+  since: "3.5.0"
+
+- name: "hll_union_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.HllUnionAgg"
+  signatures:
+    - "hll_union_agg(expr, allowDifferentLgConfigK)"
+  usage: |2-
+    
+        hll_union_agg(expr, allowDifferentLgConfigK) - Returns the estimated number of unique values.
+          `allowDifferentLgConfigK` (optional) Allow sketches with different lgConfigK values
+           to be unioned (defaults to false).
+  examples: |2-
+    
+        Examples:
+          > SELECT hll_sketch_estimate(hll_union_agg(sketch, true)) FROM (SELECT hll_sketch_agg(col) as sketch FROM VALUES (1) tab(col) UNION ALL SELECT hll_sketch_agg(col, 20) as sketch FROM VALUES (1) tab(col));
+           1
+      
+  since: "3.5.0"
+
+- name: "kll_merge_agg_bigint"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.KllMergeAggBigint"
+  signatures:
+    - "kll_merge_agg_bigint(expr[, k])"
+  usage: |2-
+    
+        kll_merge_agg_bigint(expr[, k]) - Merges binary KllLongsSketch representations and returns the merged sketch.
+          The input expression should contain binary sketch representations (e.g., from kll_sketch_agg_bigint).
+          The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535).
+          If k is not specified, the merged sketch adopts the k value from the first input sketch.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT kll_sketch_get_n_bigint(kll_merge_agg_bigint(sketch)) FROM (SELECT kll_sketch_agg_bigint(col) as sketch FROM VALUES (1), (2), (3) tab(col) UNION ALL SELECT kll_sketch_agg_bigint(col) as sketch FROM VALUES (4), (5), (6) tab(col)) t;
+           6
+      
+  since: "4.1.2"
+
+- name: "kll_merge_agg_double"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.KllMergeAggDouble"
+  signatures:
+    - "kll_merge_agg_double(expr[, k])"
+  usage: |2-
+    
+        kll_merge_agg_double(expr[, k]) - Merges binary KllDoublesSketch representations and returns the merged sketch.
+          The input expression should contain binary sketch representations (e.g., from kll_sketch_agg_double).
+          The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535).
+          If k is not specified, the merged sketch adopts the k value from the first input sketch.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT kll_sketch_get_n_double(kll_merge_agg_double(sketch)) FROM (SELECT kll_sketch_agg_double(col) as sketch FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)) tab(col) UNION ALL SELECT kll_sketch_agg_double(col) as sketch FROM VALUES (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)), (CAST(6.0 AS DOUBLE)) tab(col)) t;
+           6
+      
+  since: "4.1.2"
+
+- name: "kll_merge_agg_float"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.KllMergeAggFloat"
+  signatures:
+    - "kll_merge_agg_float(expr[, k])"
+  usage: |2-
+    
+        kll_merge_agg_float(expr[, k]) - Merges binary KllFloatsSketch representations and returns the merged sketch.
+          The input expression should contain binary sketch representations (e.g., from kll_sketch_agg_float).
+          The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535).
+          If k is not specified, the merged sketch adopts the k value from the first input sketch.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT kll_sketch_get_n_float(kll_merge_agg_float(sketch)) FROM (SELECT kll_sketch_agg_float(col) as sketch FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)) tab(col) UNION ALL SELECT kll_sketch_agg_float(col) as sketch FROM VALUES (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)), (CAST(6.0 AS FLOAT)) tab(col)) t;
+           6
+      
+  since: "4.1.2"
+
+- name: "kurtosis"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Kurtosis"
+  signatures:
+    - "kurtosis(expr)"
+  usage: "kurtosis(expr) - Returns the kurtosis value calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);
+           -0.7014368047529627
+          > SELECT kurtosis(col) FROM VALUES (1), (10), (100), (10), (1) as tab(col);
+           0.19432323191699075
+      
+  since: "1.6.0"
+
+- name: "last"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Last"
+  signatures:
+    - "last(expr[, isIgnoreNull])"
+  usage: |2-
+    
+        last(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows.
+          If `isIgnoreNull` is true, returns only non-null values
+  examples: |2-
+    
+        Examples:
+          > SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col);
+           20
+          > SELECT last(col) FROM VALUES (10), (5), (NULL) AS tab(col);
+           NULL
+          > SELECT last(col, true) FROM VALUES (10), (5), (NULL) AS tab(col);
+           5
+      
+  note: |2-
+    
+        The function is non-deterministic because its results depends on the order of the rows
+        which may be non-deterministic after a shuffle.
+      
+  since: "2.0.0"
+
+- name: "last_value"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Last"
+  signatures:
+    - "last_value(expr[, isIgnoreNull])"
+  usage: |2-
+    
+        last_value(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows.
+          If `isIgnoreNull` is true, returns only non-null values
+  examples: |2-
+    
+        Examples:
+          > SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col);
+           20
+          > SELECT last_value(col) FROM VALUES (10), (5), (NULL) AS tab(col);
+           NULL
+          > SELECT last_value(col, true) FROM VALUES (10), (5), (NULL) AS tab(col);
+           5
+      
+  note: |2-
+    
+        The function is non-deterministic because its results depends on the order of the rows
+        which may be non-deterministic after a shuffle.
+      
+  since: "2.0.0"
+
+- name: "listagg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ListAgg"
+  signatures:
+    - "listagg(expr[, delimiter])[ WITHIN GROUP (ORDER BY key [ASC | DESC] [,...])]"
+  usage: |2-
+    
+        listagg(expr[, delimiter])[ WITHIN GROUP (ORDER BY key [ASC | DESC] [,...])] - Returns
+        the concatenation of non-NULL input values, separated by the delimiter ordered by key.
+        If all values are NULL, NULL is returned.
+        
+  arguments: |2-
+    
+        Arguments:
+          * expr - a string or binary expression to be concatenated.
+          * delimiter - an optional string or binary foldable expression used to separate the input values.
+            If NULL, the concatenation will be performed without a delimiter. Default is NULL.
+          * key - an optional expression for ordering the input values. Multiple keys can be specified.
+            If none are specified, the order of the rows in the result is non-deterministic.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT listagg(col) FROM VALUES ('a'), ('b'), ('c') AS tab(col);
+           abc
+          > SELECT listagg(col) WITHIN GROUP (ORDER BY col DESC) FROM VALUES ('a'), ('b'), ('c') AS tab(col);
+           cba
+          > SELECT listagg(col) FROM VALUES ('a'), (NULL), ('b') AS tab(col);
+           ab
+          > SELECT listagg(col) FROM VALUES ('a'), ('a') AS tab(col);
+           aa
+          > SELECT listagg(DISTINCT col) FROM VALUES ('a'), ('a'), ('b') AS tab(col);
+           ab
+          > SELECT listagg(col, ', ') FROM VALUES ('a'), ('b'), ('c') AS tab(col);
+           a, b, c
+          > SELECT listagg(col) FROM VALUES (NULL), (NULL) AS tab(col);
+           NULL
+      
+  note: |2-
+    
+        * If the order is not specified, the function is non-deterministic because
+        the order of the rows may be non-deterministic after a shuffle.
+        * If DISTINCT is specified, then expr and key must be the same expression.
+      
+  since: "4.0.0"
+
+- name: "max"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Max"
+  signatures:
+    - "max(expr)"
+  usage: "max(expr) - Returns the maximum value of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col);
+           50
+      
+  since: "1.0.0"
+
+- name: "max_by"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.MaxByBuilder"
+  signatures:
+    - "max_by(x, y)"
+    - "max_by(x, y, k)"
+  usage: |2-
+    
+        max_by(x, y) - Returns the value of `x` associated with the maximum value of `y`.
+        max_by(x, y, k) - Returns an array of the `k` values of `x` associated with the
+        maximum values of `y`, sorted in descending order by `y`.
+        Returns NULL if there are no non-NULL ordering values.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT max_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);
+           b
+          > SELECT max_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);
+           ["b","c"]
+      
+  note: |2-
+    
+        The function is non-deterministic so the output order can be different for
+        those associated the same values of `y`.
+
+        The maximum value of `k` is 100000.
+      
+  since: "4.2.0"
+
+- name: "mean"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Average"
+  signatures:
+    - "mean(expr)"
+  usage: "mean(expr) - Returns the mean calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col);
+           2.0
+          > SELECT mean(col) FROM VALUES (1), (2), (NULL) AS tab(col);
+           1.5
+      
+  since: "1.0.0"
+
+- name: "measure"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Measure"
+  signatures:
+    - "measure(expr)"
+  usage: "measure(expr) - this function is used and can only be used to calculate a measure defined in a metric view."
+  examples: |2-
+    
+        Examples:
+          > SELECT dimension_col, measure(measure_col)
+            FROM test_metric_view
+            GROUP BY dimension_col;
+          dim_1, 100
+          dim_2, 200
+      
+  since: "4.2.0"
+
+- name: "median"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Median"
+  signatures:
+    - "median(col)"
+  usage: "median(col) - Returns the median of numeric or ANSI interval column `col`."
+  examples: |2-
+    
+        Examples:
+          > SELECT median(col) FROM VALUES (0), (10) AS tab(col);
+           5.0
+          > SELECT median(col) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-5
+      
+  since: "3.4.0"
+
+- name: "min"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Min"
+  signatures:
+    - "min(expr)"
+  usage: "min(expr) - Returns the minimum value of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col);
+           -1
+      
+  since: "1.0.0"
+
+- name: "min_by"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.MinByBuilder"
+  signatures:
+    - "min_by(x, y)"
+    - "min_by(x, y, k)"
+  usage: |2-
+    
+        min_by(x, y) - Returns the value of `x` associated with the minimum value of `y`.
+        min_by(x, y, k) - Returns an array of the `k` values of `x` associated with the
+        minimum values of `y`, sorted in ascending order by `y`.
+        Returns NULL if there are no non-NULL ordering values.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT min_by(x, y) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);
+           a
+          > SELECT min_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);
+           ["a","c"]
+      
+  note: |2-
+    
+        The function is non-deterministic so the output order can be different for
+        those associated the same values of `y`.
+
+        The maximum value of `k` is 100000.
+      
+  since: "4.2.0"
+
+- name: "mode"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ModeBuilder"
+  signatures:
+    - "mode(col[, deterministic])"
+    - "mode() WITHIN GROUP (ORDER BY col)"
+  usage: |2-
+    
+        mode(col[, deterministic]) - Returns the most frequent value for the values within `col`. NULL values are ignored. If all the values are NULL, or there are 0 rows, returns NULL.
+          When multiple values have the same greatest frequency then either any of values is returned if `deterministic` is false or is not defined, or the lowest value is returned if `deterministic` is true.
+        mode() WITHIN GROUP (ORDER BY col) - Returns the most frequent value for the values within `col` (specified in ORDER BY clause). NULL values are ignored.
+          If all the values are NULL, or there are 0 rows, returns NULL. When multiple values have the same greatest frequency only one value will be returned.
+          The value will be chosen based on sort direction. Return the smallest value if sort direction is asc or the largest value if sort direction is desc from multiple values with the same frequency.
+  examples: |2-
+    
+        Examples:
+          > SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col);
+           10
+          > SELECT mode(col) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '10' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-10
+          > SELECT mode(col) FROM VALUES (0), (10), (10), (null), (null), (null) AS tab(col);
+           10
+          > SELECT mode(col, false) FROM VALUES (-10), (0), (10) AS tab(col);
+           0
+          > SELECT mode(col, true) FROM VALUES (-10), (0), (10) AS tab(col);
+           -10
+          > SELECT mode() WITHIN GROUP (ORDER BY col) FROM VALUES (0), (10), (10) AS tab(col);
+           10
+          > SELECT mode() WITHIN GROUP (ORDER BY col) FROM VALUES (0), (10), (10), (20), (20) AS tab(col);
+           10
+          > SELECT mode() WITHIN GROUP (ORDER BY col DESC) FROM VALUES (0), (10), (10), (20), (20) AS tab(col);
+           20
+      
+  since: "3.4.0"
+
+- name: "percentile"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Percentile"
+  signatures:
+    - "percentile(col, percentage [, frequency])"
+    - "percentile(col, array(percentage1 [, percentage2]...) [, frequency])"
+  usage: |2-
+    
+          percentile(col, percentage [, frequency]) - Returns the exact percentile value of numeric
+           or ANSI interval column `col` at the given percentage. The value of percentage must be
+           between 0.0 and 1.0. The value of frequency should be positive integral
+    
+          percentile(col, array(percentage1 [, percentage2]...) [, frequency]) - Returns the exact
+          percentile value array of numeric column `col` at the given percentage(s). Each value
+          of the percentage array must be between 0.0 and 1.0. The value of frequency should be
+          positive integral
+    
+          
+  examples: |2-
+    
+        Examples:
+          > SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col);
+           3.0
+          > SELECT percentile(col, array(0.25, 0.75)) FROM VALUES (0), (10) AS tab(col);
+           [2.5,7.5]
+          > SELECT percentile(col, 0.5) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-5
+          > SELECT percentile(col, array(0.2, 0.5)) FROM VALUES (INTERVAL '0' SECOND), (INTERVAL '10' SECOND) AS tab(col);
+           [0 00:00:02.000000000,0 00:00:05.000000000]
+      
+  since: "2.1.0"
+
+- name: "percentile_approx"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile"
+  signatures:
+    - "percentile_approx(col, percentage [, accuracy])"
+  usage: |2-
+    
+        percentile_approx(col, percentage [, accuracy]) - Returns the approximate `percentile` of the numeric or
+          ansi interval column `col` which is the smallest value in the ordered `col` values (sorted
+          from least to greatest) such that no more than `percentage` of `col` values is less than
+          the value or equal to that value. The value of percentage must be between 0.0 and 1.0.
+          The `accuracy` parameter (default: 10000) is a positive numeric literal which controls
+          approximation accuracy at the cost of memory. Higher value of `accuracy` yields better
+          accuracy, `1.0/accuracy` is the relative error of the approximation.
+          When `percentage` is an array, each value of the percentage array must be between 0.0 and 1.0.
+          In this case, returns the approximate percentile array of column `col` at the given
+          percentage array.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);
+           [1,1,0]
+          > SELECT percentile_approx(col, 0.5, 100) FROM VALUES (0), (6), (7), (9), (10) AS tab(col);
+           7
+          > SELECT percentile_approx(col, 0.5, 100) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '1' MONTH), (INTERVAL '2' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-1
+          > SELECT percentile_approx(col, array(0.5, 0.7), 100) FROM VALUES (INTERVAL '0' SECOND), (INTERVAL '1' SECOND), (INTERVAL '2' SECOND), (INTERVAL '10' SECOND) AS tab(col);
+           [0 00:00:01.000000000,0 00:00:02.000000000]
+      
+  since: "2.1.0"
+
+- name: "percentile_cont"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.PercentileContBuilder"
+  signatures:
+    - "percentile_cont(percentage) WITHIN GROUP (ORDER BY col)"
+  usage: "percentile_cont(percentage) WITHIN GROUP (ORDER BY col) - Return a percentile value based on a continuous distribution of numeric or ANSI interval column `col` at the given `percentage` (specified in ORDER BY clause)."
+  examples: |2-
+    
+        Examples:
+          > SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY col) FROM VALUES (0), (10) AS tab(col);
+           2.5
+          > SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY col) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-2
+      
+  since: "4.0.0"
+
+- name: "percentile_disc"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.PercentileDiscBuilder"
+  signatures:
+    - "percentile_disc(percentage) WITHIN GROUP (ORDER BY col)"
+  usage: "percentile_disc(percentage) WITHIN GROUP (ORDER BY col) - Return a percentile value based on a discrete distribution of numeric or ANSI interval column `col` at the given `percentage` (specified in ORDER BY clause)."
+  examples: |2-
+    
+        Examples:
+          > SELECT percentile_disc(0.25) WITHIN GROUP (ORDER BY col) FROM VALUES (0), (10) AS tab(col);
+           0.0
+          > SELECT percentile_disc(0.25) WITHIN GROUP (ORDER BY col) FROM VALUES (INTERVAL '0' MONTH), (INTERVAL '10' MONTH) AS tab(col);
+           0-0
+      
+  since: "4.0.0"
+
+- name: "product"
+  scope: "spark_internal"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Product"
+  signatures:
+    - "product(expr)"
+  usage: "product(expr) - Returns the product of non-null numeric values in a group."
+
+- name: "regr_avgx"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrAvgX"
+  signatures:
+    - "regr_avgx(y, x)"
+  usage: "regr_avgx(y, x) - Returns the average of the independent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           2.75
+          > SELECT regr_avgx(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_avgx(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           3.0
+          > SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           3.0
+      
+  since: "3.3.0"
+
+- name: "regr_avgy"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrAvgY"
+  signatures:
+    - "regr_avgy(y, x)"
+  usage: "regr_avgy(y, x) - Returns the average of the dependent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           1.75
+          > SELECT regr_avgy(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_avgy(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           1.6666666666666667
+          > SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           1.5
+      
+  since: "3.3.0"
+
+- name: "regr_count"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrCount"
+  signatures:
+    - "regr_count(y, x)"
+  usage: "regr_count(y, x) - Returns the number of non-null number pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           4
+          > SELECT regr_count(y, x) FROM VALUES (1, null) AS tab(y, x);
+           0
+          > SELECT regr_count(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           0
+          > SELECT regr_count(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           3
+          > SELECT regr_count(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           2
+      
+  since: "3.3.0"
+
+- name: "regr_intercept"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrIntercept"
+  signatures:
+    - "regr_intercept(y, x)"
+  usage: "regr_intercept(y, x) - Returns the intercept of the univariate linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_intercept(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x);
+           0.0
+          > SELECT regr_intercept(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_intercept(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_intercept(y, x) FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x);
+           0.0
+          > SELECT regr_intercept(y, x) FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x);
+           0.0
+      
+  since: "3.4.0"
+
+- name: "regr_r2"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrR2"
+  signatures:
+    - "regr_r2(y, x)"
+  usage: "regr_r2(y, x) - Returns the coefficient of determination for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           0.2727272727272727
+          > SELECT regr_r2(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_r2(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           0.7500000000000001
+          > SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           1.0
+      
+  since: "3.3.0"
+
+- name: "regr_slope"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrSlope"
+  signatures:
+    - "regr_slope(y, x)"
+  usage: "regr_slope(y, x) - Returns the slope of the linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_slope(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x);
+           1.0
+          > SELECT regr_slope(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_slope(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_slope(y, x) FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x);
+           1.0
+          > SELECT regr_slope(y, x) FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x);
+           1.0
+      
+  since: "3.4.0"
+
+- name: "regr_sxx"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrSXX"
+  signatures:
+    - "regr_sxx(y, x)"
+  usage: "regr_sxx(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           2.75
+          > SELECT regr_sxx(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_sxx(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           2.0
+          > SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           2.0
+      
+  since: "3.4.0"
+
+- name: "regr_sxy"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrSXY"
+  signatures:
+    - "regr_sxy(y, x)"
+  usage: "regr_sxy(y, x) - Returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           0.75
+          > SELECT regr_sxy(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_sxy(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           1.0
+          > SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           1.0
+      
+  since: "3.4.0"
+
+- name: "regr_syy"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.RegrSYY"
+  signatures:
+    - "regr_syy(y, x)"
+  usage: "regr_syy(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable."
+  examples: |2-
+    
+        Examples:
+          > SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);
+           0.75
+          > SELECT regr_syy(y, x) FROM VALUES (1, null) AS tab(y, x);
+           NULL
+          > SELECT regr_syy(y, x) FROM VALUES (null, 1) AS tab(y, x);
+           NULL
+          > SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x);
+           0.6666666666666666
+          > SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x);
+           0.5
+      
+  since: "3.4.0"
+
+- name: "skewness"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Skewness"
+  signatures:
+    - "skewness(expr)"
+  usage: "skewness(expr) - Returns the skewness value calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);
+           1.1135657469022011
+          > SELECT skewness(col) FROM VALUES (-1000), (-100), (10), (20) AS tab(col);
+           -1.1135657469022011
+      
+  since: "1.6.0"
+
+- name: "some"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr"
+  signatures:
+    - "some(expr)"
+  usage: "some(expr) - Returns true if at least one value of `expr` is true."
+  examples: |2-
+    
+        Examples:
+          > SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col);
+           true
+          > SELECT some(col) FROM VALUES (NULL), (true), (false) AS tab(col);
+           true
+          > SELECT some(col) FROM VALUES (false), (false), (NULL) AS tab(col);
+           false
+      
+  since: "3.0.0"
+
+- name: "std"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp"
+  signatures:
+    - "std(expr)"
+  usage: "std(expr) - Returns the sample standard deviation calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col);
+           1.0
+      
+  since: "1.6.0"
+
+- name: "stddev"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp"
+  signatures:
+    - "stddev(expr)"
+  usage: "stddev(expr) - Returns the sample standard deviation calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col);
+           1.0
+      
+  since: "1.6.0"
+
+- name: "stddev_pop"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop"
+  signatures:
+    - "stddev_pop(expr)"
+  usage: "stddev_pop(expr) - Returns the population standard deviation calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col);
+           0.816496580927726
+      
+  since: "1.6.0"
+
+- name: "stddev_samp"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp"
+  signatures:
+    - "stddev_samp(expr)"
+  usage: "stddev_samp(expr) - Returns the sample standard deviation calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col);
+           1.0
+      
+  since: "1.6.0"
+
+- name: "string_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ListAgg"
+  signatures:
+    - "string_agg(expr[, delimiter])[ WITHIN GROUP (ORDER BY key [ASC | DESC] [,...])]"
+  usage: |2-
+    
+        string_agg(expr[, delimiter])[ WITHIN GROUP (ORDER BY key [ASC | DESC] [,...])] - Returns
+        the concatenation of non-NULL input values, separated by the delimiter ordered by key.
+        If all values are NULL, NULL is returned.
+        
+  arguments: |2-
+    
+        Arguments:
+          * expr - a string or binary expression to be concatenated.
+          * delimiter - an optional string or binary foldable expression used to separate the input values.
+            If NULL, the concatenation will be performed without a delimiter. Default is NULL.
+          * key - an optional expression for ordering the input values. Multiple keys can be specified.
+            If none are specified, the order of the rows in the result is non-deterministic.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT string_agg(col) FROM VALUES ('a'), ('b'), ('c') AS tab(col);
+           abc
+          > SELECT string_agg(col) WITHIN GROUP (ORDER BY col DESC) FROM VALUES ('a'), ('b'), ('c') AS tab(col);
+           cba
+          > SELECT string_agg(col) FROM VALUES ('a'), (NULL), ('b') AS tab(col);
+           ab
+          > SELECT string_agg(col) FROM VALUES ('a'), ('a') AS tab(col);
+           aa
+          > SELECT string_agg(DISTINCT col) FROM VALUES ('a'), ('a'), ('b') AS tab(col);
+           ab
+          > SELECT string_agg(col, ', ') FROM VALUES ('a'), ('b'), ('c') AS tab(col);
+           a, b, c
+          > SELECT string_agg(col) FROM VALUES (NULL), (NULL) AS tab(col);
+           NULL
+      
+  note: |2-
+    
+        * If the order is not specified, the function is non-deterministic because
+        the order of the rows may be non-deterministic after a shuffle.
+        * If DISTINCT is specified, then expr and key must be the same expression.
+      
+  since: "4.0.0"
+
+- name: "sum"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.Sum"
+  signatures:
+    - "sum(expr)"
+  usage: "sum(expr) - Returns the sum calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col);
+           30
+          > SELECT sum(col) FROM VALUES (NULL), (10), (15) AS tab(col);
+           25
+          > SELECT sum(col) FROM VALUES (NULL), (NULL) AS tab(col);
+           NULL
+      
+  since: "1.0.0"
+
+- name: "theta_intersection_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ThetaIntersectionAgg"
+  signatures:
+    - "theta_intersection_agg(expr)"
+  usage: |2-
+    
+        theta_intersection_agg(expr) - Returns the ThetaSketch's Compact binary representation
+          by intersecting all the Theta sketches in the input column.
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_intersection_agg(sketch)) FROM (SELECT theta_sketch_agg(col) as sketch FROM VALUES (1) tab(col) UNION ALL SELECT theta_sketch_agg(col, 20) as sketch FROM VALUES (1) tab(col));
+           1
+      
+  since: "4.1.0"
+
+- name: "theta_sketch_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ThetaSketchAgg"
+  signatures:
+    - "theta_sketch_agg(expr, lgNomEntries)"
+  usage: |2-
+    
+        theta_sketch_agg(expr, lgNomEntries) - Returns the ThetaSketch compact binary representation.
+          `lgNomEntries` (optional) is the log-base-2 of nominal entries, with nominal entries deciding
+          the number buckets or slots for the ThetaSketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_sketch_agg(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col);
+           3
+      
+  since: "4.1.0"
+
+- name: "theta_union_agg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.ThetaUnionAgg"
+  signatures:
+    - "theta_union_agg(expr, lgNomEntries)"
+  usage: |2-
+    
+        theta_union_agg(expr, lgNomEntries) - Returns the ThetaSketch's Compact binary representation.
+          `lgNomEntries` (optional) the log-base-2 of Nominal Entries, with Nominal Entries deciding
+          the number buckets or slots for the ThetaSketch.
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_union_agg(sketch)) FROM (SELECT theta_sketch_agg(col) as sketch FROM VALUES (1) tab(col) UNION ALL SELECT theta_sketch_agg(col, 20) as sketch FROM VALUES (1) tab(col));
+           1
+      
+  since: "4.1.0"
+
+- name: "try_avg"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TryAverageExpressionBuilder"
+  signatures:
+    - "try_avg(expr)"
+  usage: "try_avg(expr) - Returns the mean calculated from values of a group and the result is null on overflow."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col);
+           2.0
+          > SELECT try_avg(col) FROM VALUES (1), (2), (NULL) AS tab(col);
+           1.5
+          > SELECT try_avg(col) FROM VALUES (interval '2147483647 months'), (interval '1 months') AS tab(col);
+           NULL
+      
+  since: "3.3.0"
+
+- name: "try_sum"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TrySumExpressionBuilder"
+  signatures:
+    - "try_sum(expr)"
+  usage: "try_sum(expr) - Returns the sum calculated from values of a group and the result is null on overflow."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col);
+           30
+          > SELECT try_sum(col) FROM VALUES (NULL), (10), (15) AS tab(col);
+           25
+          > SELECT try_sum(col) FROM VALUES (NULL), (NULL) AS tab(col);
+           NULL
+          > SELECT try_sum(col) FROM VALUES (9223372036854775807L), (1L) AS tab(col);
+           NULL
+      
+  since: "3.3.0"
+
+- name: "tuple_intersection_agg_double"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TupleIntersectionAggDouble"
+  signatures:
+    - "tuple_intersection_agg_double(child, mode)"
+  usage: |2-
+    
+        tuple_intersection_agg_double(child, mode) - Returns the intersected TupleSketch compact binary representation.
+          `child` should be a binary TupleSketch representation created with a double type summary.
+          `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D), (3, 15.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D), (4, 12.0D) tab(key, summary));
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_intersection_agg_integer"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TupleIntersectionAggInteger"
+  signatures:
+    - "tuple_intersection_agg_integer(child, mode)"
+  usage: |2-
+    
+        tuple_intersection_agg_integer(child, mode) - Returns the intersected TupleSketch compact binary representation.
+          `child` should be a binary TupleSketch representation created with an integer type summary.
+          `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 2), (3, 3), (4, 4) tab(key, summary));
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_agg_double"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TupleSketchAggDoubleExpressionBuilder"
+  signatures:
+    - "tuple_sketch_agg_double(key, summary, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_sketch_agg_double(key, summary, lgNomEntries, mode) - Returns the TupleSketch compact binary representation.
+          `key` is the expression for unique value counting.
+          `summary` is the double value to be aggregated.
+          `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding
+          the number buckets or slots for the TupleSketch. Default is 12.
+          `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, 'sum')) FROM VALUES (1, 5.0D), (1, 1.0D), (2, 2.0D), (2, 3.0D), (3, 2.2D) tab(key, summary);
+           3.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_agg_integer"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TupleSketchAggIntegerExpressionBuilder"
+  signatures:
+    - "tuple_sketch_agg_integer(key, summary, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_sketch_agg_integer(key, summary, lgNomEntries, mode) - Returns the TupleSketch compact binary representation.
+          `key` is the expression for unique value counting.
+          `summary` is the integer value to be aggregated.
+          `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding
+          the number buckets or slots for the TupleSketch. Default is 12.
+          `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, 'sum')) FROM VALUES (1, 5), (1, 1), (2, 2), (2, 3), (3, 2) tab(key, summary);
+           3.0
+      
+  since: "4.2.0"
+
+- name: "tuple_union_agg_double"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TupleUnionAggDoubleExpressionBuilder"
+  signatures:
+    - "tuple_union_agg_double(child, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_union_agg_double(child, lgNomEntries, mode) - Returns the unioned TupleSketch compact binary representation.
+          `child` should be a binary TupleSketch representation created with a double type summary.
+          `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12.
+          `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_union_agg_double(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D) tab(key, summary));
+           3.0
+      
+  since: "4.2.0"
+
+- name: "tuple_union_agg_integer"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.TupleUnionAggIntegerExpressionBuilder"
+  signatures:
+    - "tuple_union_agg_integer(child, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_union_agg_integer(child, lgNomEntries, mode) - Returns the unioned TupleSketch compact binary representation.
+          `child` should be a binary TupleSketch representation created with an integer type summary.
+          `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12.
+          `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 5), (2, 10) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 3), (3, 7) tab(key, summary));
+           3.0
+      
+  since: "4.2.0"
+
+- name: "var_pop"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.VariancePop"
+  signatures:
+    - "var_pop(expr)"
+  usage: "var_pop(expr) - Returns the population variance calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col);
+           0.6666666666666666
+      
+  since: "1.6.0"
+
+- name: "var_samp"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp"
+  signatures:
+    - "var_samp(expr)"
+  usage: "var_samp(expr) - Returns the sample variance calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col);
+           1.0
+      
+  since: "1.6.0"
+
+- name: "variance"
+  scope: "sql_builtin"
+  group: "agg_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp"
+  signatures:
+    - "variance(expr)"
+  usage: "variance(expr) - Returns the sample variance calculated from values of a group."
+  examples: |2-
+    
+        Examples:
+          > SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col);
+           1.0
+      
+  since: "1.6.0"
+
+- name: "vector_avg"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorAvg"
+  signatures:
+    - "vector_avg(array)"
+  usage: |2-
+    
+        vector_avg(array) - Returns the element-wise mean of float vectors in a group.
+        All vectors must have the same dimension.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_avg(col) FROM VALUES (array(1.0F, 2.0F)), (array(3.0F, 4.0F)) AS tab(col);
+           [2.0,3.0]
+      
+  since: "4.2.0"
+
+- name: "vector_sum"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorSum"
+  signatures:
+    - "vector_sum(array)"
+  usage: |2-
+    
+        vector_sum(array) - Returns the element-wise sum of float vectors in a group.
+        All vectors must have the same dimension.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_sum(col) FROM VALUES (array(1.0F, 2.0F)), (array(3.0F, 4.0F)) AS tab(col);
+           [4.0,6.0]
+      
+  since: "4.2.0"
diff --git a/crates/sail-plan/data/functions/generator.yaml b/crates/sail-plan/data/functions/generator.yaml
new file mode 100644
index 000000000..ebfccf392
--- /dev/null
+++ b/crates/sail-plan/data/functions/generator.yaml
@@ -0,0 +1,177 @@
+- name: "explode"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ExplodeGeneratorBuilder"
+  signatures:
+    - "explode(expr)"
+  usage: "explode(expr) - Separates the elements of array `expr` into multiple rows, or the elements of map `expr` into multiple rows and columns. Unless specified otherwise, uses the default column name `col` for elements of the array or `key` and `value` for the elements of the map."
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM explode(array(10, 20));
+           10
+           20
+          > SELECT * FROM explode(collection => array(10, 20));
+           10
+           20
+      
+  since: "3.4.0"
+
+- name: "explode_outer"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ExplodeOuterGeneratorBuilder"
+  signatures:
+    - "explode_outer(expr)"
+  usage: "explode_outer(expr) - Separates the elements of array `expr` into multiple rows, or the elements of map `expr` into multiple rows and columns. Unless specified otherwise, uses the default column name `col` for elements of the array or `key` and `value` for the elements of the map."
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM explode_outer(array(10, 20));
+           10
+           20
+          > SELECT * FROM explode_outer(collection => array(10, 20));
+           10
+           20
+      
+  since: "3.4.0"
+
+- name: "inline"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.InlineGeneratorBuilder"
+  signatures:
+    - "inline(expr)"
+  usage: "inline(expr) - Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b')));
+           1	a
+           2	b
+          > SELECT * FROM inline(input => array(struct(1, 'a'), struct(2, 'b')));
+           1	a
+           2	b
+      
+  since: "3.4.0"
+
+- name: "inline_outer"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.InlineOuterGeneratorBuilder"
+  signatures:
+    - "inline_outer(expr)"
+  usage: "inline_outer(expr) - Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM inline_outer(array(struct(1, 'a'), struct(2, 'b')));
+           1	a
+           2	b
+          > SELECT * FROM inline_outer(input => array(struct(1, 'a'), struct(2, 'b')));
+           1	a
+           2	b
+      
+  since: "3.4.0"
+
+- name: "posexplode"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.PosExplodeGeneratorBuilder"
+  signatures:
+    - "posexplode(expr)"
+  usage: "posexplode(expr) - Separates the elements of array `expr` into multiple rows with positions, or the elements of map `expr` into multiple rows and columns with positions. Unless specified otherwise, uses the column name `pos` for position, `col` for elements of the array or `key` and `value` for elements of the map."
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM posexplode(array(10,20));
+           0	10
+           1	20
+          > SELECT * FROM posexplode(collection => array(10,20));
+           0	10
+           1	20
+      
+  since: "3.5.0"
+
+- name: "posexplode_outer"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.PosExplodeOuterGeneratorBuilder"
+  signatures:
+    - "posexplode_outer(expr)"
+  usage: "posexplode_outer(expr) - Separates the elements of array `expr` into multiple rows with positions, or the elements of map `expr` into multiple rows and columns with positions. Unless specified otherwise, uses the column name `pos` for position, `col` for elements of the array or `key` and `value` for elements of the map."
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM posexplode_outer(array(10,20));
+           0	10
+           1	20
+          > SELECT * FROM posexplode_outer(collection => array(10,20));
+           0	10
+           1	20
+      
+  since: "3.5.0"
+
+- name: "stack"
+  scope: "sql_builtin"
+  group: "generator_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Stack"
+  signatures:
+    - "stack(n, expr1, ..., exprk)"
+  usage: "stack(n, expr1, ..., exprk) - Separates `expr1`, ..., `exprk` into `n` rows. Uses column names col0, col1, etc. by default unless specified otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT stack(2, 1, 2, 3);
+           1	2
+           3	NULL
+      
+  since: "2.0.0"
+
+- name: "variant_explode"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.VariantExplodeGeneratorBuilder"
+  signatures:
+    - "variant_explode(expr)"
+  usage: "variant_explode(expr) - It separates a variant object/array into multiple rows containing its fields/elements. Its result schema is `struct`. `pos` is the position of the field/element in its parent object/array, and `value` is the field/element value. `key` is the field name when exploding a variant object, or is NULL when exploding a variant array. It ignores any input that is not a variant array/object, including SQL NULL, variant null, and any other variant values."
+  examples: |2-
+    
+        Examples:
+          > SELECT * from variant_explode(parse_json('["hello", "world"]'));
+           0	NULL	"hello"
+           1	NULL	"world"
+          > SELECT * from variant_explode(input => parse_json('{"a": true, "b": 3.14}'));
+           0	a	true
+           1	b	3.14
+      
+  since: "4.0.0"
+
+- name: "variant_explode_outer"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.VariantExplodeOuterGeneratorBuilder"
+  signatures:
+    - "variant_explode_outer(expr)"
+  usage: "variant_explode_outer(expr) - It separates a variant object/array into multiple rows containing its fields/elements. Its result schema is `struct`. `pos` is the position of the field/element in its parent object/array, and `value` is the field/element value. `key` is the field name when exploding a variant object, or is NULL when exploding a variant array. It ignores any input that is not a variant array/object, including SQL NULL, variant null, and any other variant values."
+  examples: |2-
+    
+        Examples:
+          > SELECT * from variant_explode_outer(parse_json('["hello", "world"]'));
+           0	NULL	"hello"
+           1	NULL	"world"
+          > SELECT * from variant_explode_outer(input => parse_json('{"a": true, "b": 3.14}'));
+           0	a	true
+           1	b	3.14
+      
+  since: "4.0.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/array.yaml b/crates/sail-plan/data/functions/scalar/array.yaml
new file mode 100644
index 000000000..d16a45c14
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/array.yaml
@@ -0,0 +1,535 @@
+- name: "array"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CreateArray"
+  signatures:
+    - "array(expr, ...)"
+  usage: "array(expr, ...) - Returns an array with the given elements."
+  examples: |2-
+    
+        Examples:
+          > SELECT array(1, 2, 3);
+           [1,2,3]
+      
+  since: "1.1.0"
+
+- name: "array_append"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayAppend"
+  signatures:
+    - "array_append(array, element)"
+  usage: |2-
+    
+          array_append(array, element) - Add the element at the end of the array passed as first
+          argument. Type of element should be similar to type of the elements of the array.
+          Null element is also appended into the array. But if the array passed, is NULL
+          output is NULL
+          
+  examples: |2-
+    
+        Examples:
+          > SELECT array_append(array('b', 'd', 'c', 'a'), 'd');
+           ["b","d","c","a","d"]
+          > SELECT array_append(array(1, 2, 3, null), null);
+           [1,2,3,null,null]
+          > SELECT array_append(CAST(null as Array), 2);
+           NULL
+      
+  since: "3.4.0"
+
+- name: "array_compact"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayCompact"
+  signatures:
+    - "array_compact(array)"
+  usage: "array_compact(array) - Removes null values from the array."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_compact(array(1, 2, 3, null));
+           [1,2,3]
+          > SELECT array_compact(array("a", "b", "c"));
+           ["a","b","c"]
+      
+  since: "3.4.0"
+
+- name: "array_concat"
+  scope: "sail_extension"
+  signatures:
+    - "array_concat(array, ...)"
+  usage: "array_concat(array, ...) - Concatenates input arrays and returns the resulting array."
+
+- name: "array_contains"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayContains"
+  signatures:
+    - "array_contains(array, value)"
+  usage: "array_contains(array, value) - Returns true if the array contains the value."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_contains(array(1, 2, 3), 2);
+           true
+      
+  since: "1.5.0"
+
+- name: "array_contains_all"
+  scope: "sail_extension"
+  signatures:
+    - "array_contains_all(array, values)"
+  usage: "array_contains_all(array, values) - Returns true if array contains every element in values."
+
+- name: "array_distinct"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayDistinct"
+  signatures:
+    - "array_distinct(array)"
+  usage: "array_distinct(array) - Removes duplicate values from the array."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_distinct(array(1, 2, 3, null, 3));
+           [1,2,3,null]
+      
+  since: "2.4.0"
+
+- name: "array_except"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayExcept"
+  signatures:
+    - "array_except(array1, array2)"
+  usage: |2-
+    
+      array_except(array1, array2) - Returns an array of the elements in array1 but not in array2,
+        without duplicates.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT array_except(array(1, 2, 3), array(1, 3, 5));
+           [2]
+      
+  since: "2.4.0"
+
+- name: "array_insert"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayInsert"
+  signatures:
+    - "array_insert(x, pos, val)"
+  usage: |2-
+    
+        array_insert(x, pos, val) - Places val into index pos of array x.
+          Array indices start at 1. The maximum negative index is -1 for which the function inserts
+          new element after the current last element.
+          Index above array size appends the array, or prepends the array if index is negative,
+          with 'null' elements.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT array_insert(array(1, 2, 3, 4), 5, 5);
+           [1,2,3,4,5]
+          > SELECT array_insert(array(5, 4, 3, 2), -1, 1);
+           [5,4,3,2,1]
+          > SELECT array_insert(array(5, 3, 2, 1), -4, 4);
+           [5,4,3,2,1]
+      
+  since: "3.4.0"
+
+- name: "array_intersect"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayIntersect"
+  signatures:
+    - "array_intersect(array1, array2)"
+  usage: |2-
+    
+      array_intersect(array1, array2) - Returns an array of the elements in the intersection of array1 and
+        array2, without duplicates.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT array_intersect(array(1, 2, 3), array(1, 3, 5));
+           [1,3]
+      
+  since: "2.4.0"
+
+- name: "array_join"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayJoin"
+  signatures:
+    - "array_join(array, delimiter[, nullReplacement])"
+  usage: |2-
+    
+        array_join(array, delimiter[, nullReplacement]) - Concatenates the elements of the given array
+          using the delimiter and an optional string to replace nulls. If no value is set for
+          nullReplacement, any null value is filtered.
+  examples: |2-
+    
+        Examples:
+          > SELECT array_join(array('hello', 'world'), ' ');
+           hello world
+          > SELECT array_join(array('hello', null ,'world'), ' ');
+           hello world
+          > SELECT array_join(array('hello', null ,'world'), ' ', ',');
+           hello , world
+      
+  since: "2.4.0"
+
+- name: "array_max"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayMax"
+  signatures:
+    - "array_max(array)"
+  usage: |2-
+    
+        array_max(array) - Returns the maximum value in the array. NaN is greater than
+        any non-NaN elements for double/float type. NULL elements are skipped.
+  examples: |2-
+    
+        Examples:
+          > SELECT array_max(array(1, 20, null, 3));
+           20
+      
+  since: "2.4.0"
+
+- name: "array_min"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayMin"
+  signatures:
+    - "array_min(array)"
+  usage: |2-
+    
+        array_min(array) - Returns the minimum value in the array. NaN is greater than
+        any non-NaN elements for double/float type. NULL elements are skipped.
+  examples: |2-
+    
+        Examples:
+          > SELECT array_min(array(1, 20, null, 3));
+           1
+      
+  since: "2.4.0"
+
+- name: "array_position"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayPosition"
+  signatures:
+    - "array_position(array, element)"
+  usage: |2-
+    
+        array_position(array, element) - Returns the (1-based) index of the first matching element of
+          the array as long, or 0 if no match is found.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT array_position(array(312, 773, 708, 708), 708);
+           3
+          > SELECT array_position(array(312, 773, 708, 708), 414);
+           0
+      
+  since: "2.4.0"
+
+- name: "array_prepend"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayPrepend"
+  signatures:
+    - "array_prepend(array, element)"
+  usage: |2-
+    
+          array_prepend(array, element) - Add the element at the beginning of the array passed as first
+          argument. Type of element should be the same as the type of the elements of the array.
+          Null element is also prepended to the array. But if the array passed is NULL
+          output is NULL
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT array_prepend(array('b', 'd', 'c', 'a'), 'd');
+           ["d","b","d","c","a"]
+          > SELECT array_prepend(array(1, 2, 3, null), null);
+           [null,1,2,3,null]
+          > SELECT array_prepend(CAST(null as Array), 2);
+           NULL
+      
+  since: "3.5.0"
+
+- name: "array_remove"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayRemove"
+  signatures:
+    - "array_remove(array, element)"
+  usage: "array_remove(array, element) - Remove all elements that equal to element from array."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_remove(array(1, 2, 3, null, 3), 3);
+           [1,2,null]
+      
+  since: "2.4.0"
+
+- name: "array_repeat"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayRepeat"
+  signatures:
+    - "array_repeat(element, count)"
+  usage: "array_repeat(element, count) - Returns the array containing element count times."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_repeat('123', 2);
+           ["123","123"]
+      
+  since: "2.4.0"
+
+- name: "array_size"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArraySize"
+  signatures:
+    - "array_size(expr)"
+  usage: "array_size(expr) - Returns the size of an array. The function returns null for null input."
+  examples: |2-
+    
+        Examples:
+          > SELECT array_size(array('b', 'd', 'c', 'a'));
+           4
+      
+  since: "3.3.0"
+
+- name: "array_union"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayUnion"
+  signatures:
+    - "array_union(array1, array2)"
+  usage: |2-
+    
+        array_union(array1, array2) - Returns an array of the elements in the union of array1 and array2,
+          without duplicates.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT array_union(array(1, 2, 3), array(1, 3, 5));
+           [1,2,3,5]
+      
+  since: "2.4.0"
+
+- name: "arrays_overlap"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArraysOverlap"
+  signatures:
+    - "arrays_overlap(a1, a2)"
+  usage: "arrays_overlap(a1, a2) - Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5));
+           true
+      
+  since: "2.4.0"
+
+- name: "arrays_zip"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArraysZip"
+  signatures:
+    - "arrays_zip(a1, a2, ...)"
+  usage: |2-
+    
+        arrays_zip(a1, a2, ...) - Returns a merged array of structs in which the N-th struct contains all
+        N-th values of input arrays.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4));
+           [{"0":1,"1":2},{"0":2,"1":3},{"0":3,"1":4}]
+          > SELECT arrays_zip(array(1, 2), array(2, 3), array(3, 4));
+           [{"0":1,"1":2,"2":3},{"0":2,"1":3,"2":4}]
+      
+  since: "2.4.0"
+
+- name: "flatten"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Flatten"
+  signatures:
+    - "flatten(arrayOfArrays)"
+  usage: "flatten(arrayOfArrays) - Transforms an array of arrays into a single array."
+  examples: |2-
+    
+        Examples:
+          > SELECT flatten(array(array(1, 2), array(3, 4)));
+           [1,2,3,4]
+      
+  since: "2.4.0"
+
+- name: "get"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Get"
+  signatures:
+    - "get(array, index)"
+  usage: |2-
+    
+        get(array, index) - Returns element of array at given (0-based) index. If the index points
+         outside of the array boundaries, then this function returns NULL.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT get(array(1, 2, 3), 0);
+           1
+          > SELECT get(array(1, 2, 3), 3);
+           NULL
+          > SELECT get(array(1, 2, 3), -1);
+           NULL
+      
+  since: "3.4.0"
+
+- name: "sequence"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sequence"
+  signatures:
+    - "sequence(start, stop, step)"
+  usage: |2-
+    
+        sequence(start, stop, step) - Generates an array of elements from start to stop (inclusive),
+          incrementing by step. The type of the returned elements is the same as the type of argument
+          expressions.
+    
+          Supported types are: byte, short, integer, long, date, timestamp.
+    
+          The start and stop expressions must resolve to the same type.
+          If start and stop expressions resolve to the 'date' or 'timestamp' type
+          then the step expression must resolve to the 'interval' or 'year-month interval' or
+          'day-time interval' type, otherwise to the same type as the start and stop expressions.
+      
+  arguments: |2-
+    
+        Arguments:
+          * start - an expression. The start of the range.
+          * stop - an expression. The end the range (inclusive).
+          * step - an optional expression. The step of the range.
+              By default step is 1 if start is less than or equal to stop, otherwise -1.
+              For the temporal sequences it's 1 day and -1 day respectively.
+              If start is greater than stop then the step must be negative, and vice versa.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sequence(1, 5);
+           [1,2,3,4,5]
+          > SELECT sequence(5, 1);
+           [5,4,3,2,1]
+          > SELECT sequence(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month);
+           [2018-01-01,2018-02-01,2018-03-01]
+          > SELECT sequence(to_date('2018-01-01'), to_date('2018-03-01'), interval '0-1' year to month);
+           [2018-01-01,2018-02-01,2018-03-01]
+      
+  since: "2.4.0"
+
+- name: "shuffle"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Shuffle"
+  signatures:
+    - "shuffle(array)"
+  usage: "shuffle(array) - Returns a random permutation of the given array."
+  examples: |2-
+    
+        Examples:
+          > SELECT shuffle(array(1, 20, 3, 5));
+           [3,1,5,20]
+          > SELECT shuffle(array(1, 20, null, 3));
+           [20,null,3,1]
+      
+  note: |2-
+    
+        The function is non-deterministic.
+      
+  since: "2.4.0"
+
+- name: "slice"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Slice"
+  signatures:
+    - "slice(x, start, length)"
+  usage: "slice(x, start, length) - Subsets array x starting from index start (array indices start at 1, or starting from the end if start is negative) with the specified length."
+  examples: |2-
+    
+        Examples:
+          > SELECT slice(array(1, 2, 3, 4), 2, 2);
+           [2,3]
+          > SELECT slice(array(1, 2, 3, 4), -2, 2);
+           [3,4]
+      
+  since: "2.4.0"
+
+- name: "sort_array"
+  scope: "sql_builtin"
+  group: "array_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SortArray"
+  signatures:
+    - "sort_array(array[, ascendingOrder])"
+  usage: |2-
+    
+        sort_array(array[, ascendingOrder]) - Sorts the input array in ascending or descending order
+          according to the natural ordering of the array elements. NaN is greater than any non-NaN
+          elements for double/float type. Null elements will be placed at the beginning of the returned
+          array in ascending order or at the end of the returned array in descending order.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sort_array(array('b', 'd', null, 'c', 'a'), true);
+           [null,"a","b","c","d"]
+          > SELECT sort_array(array('b', 'd', null, 'c', 'a'), false);
+           ["d","c","b","a",null]
+      
+  since: "1.5.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/bitwise.yaml b/crates/sail-plan/data/functions/scalar/bitwise.yaml
new file mode 100644
index 000000000..e2e22b53b
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/bitwise.yaml
@@ -0,0 +1,264 @@
+- name: "&"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseAnd"
+  signatures:
+    - "expr1 & expr2"
+  usage: "expr1 & expr2 - Returns the result of bitwise AND of `expr1` and `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 3 & 5;
+           1
+      
+  since: "1.4.0"
+
+- name: "<<"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ShiftLeft"
+  signatures:
+    - "base << exp"
+  usage: "base << exp - Bitwise left shift."
+  examples: |2-
+    
+        Examples:
+          > SELECT shiftleft(2, 1);
+           4
+          > SELECT 2 << 1;
+           4
+      
+  note: |2-
+    
+          `<<` operator is added in Spark 4.0.0 as an alias for `shiftleft`.
+        
+  since: "4.0.0"
+
+- name: ">>"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ShiftRight"
+  signatures:
+    - "base >> expr"
+  usage: "base >> expr - Bitwise (signed) right shift."
+  examples: |2-
+    
+        Examples:
+          > SELECT shiftright(4, 1);
+           2
+          > SELECT 4 >> 1;
+           2
+      
+  note: |2-
+    
+          `>>` operator is added in Spark 4.0.0 as an alias for `shiftright`.
+        
+  since: "4.0.0"
+
+- name: ">>>"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ShiftRightUnsigned"
+  signatures:
+    - "base >>> expr"
+  usage: "base >>> expr - Bitwise unsigned right shift."
+  examples: |2-
+    
+        Examples:
+          > SELECT shiftrightunsigned(4, 1);
+           2
+          > SELECT 4 >>> 1;
+           2
+      
+  note: |2-
+    
+        `>>>` operator is added in Spark 4.0.0 as an alias for `shiftrightunsigned`.
+      
+  since: "4.0.0"
+
+- name: "^"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseXor"
+  signatures:
+    - "expr1 ^ expr2"
+  usage: "expr1 ^ expr2 - Returns the result of bitwise exclusive OR of `expr1` and `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 3 ^ 5;
+           6
+      
+  since: "1.4.0"
+
+- name: "bit_count"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseCount"
+  signatures:
+    - "bit_count(expr)"
+  usage: "bit_count(expr) - Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, or NULL if the argument is NULL."
+  examples: |2-
+    
+        Examples:
+          > SELECT bit_count(0);
+           0
+      
+  since: "3.0.0"
+
+- name: "bit_get"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseGet"
+  signatures:
+    - "bit_get(expr, pos)"
+  usage: |2-
+    
+        bit_get(expr, pos) - Returns the value of the bit (0 or 1) at the specified position.
+          The positions are numbered from right to left, starting at zero.
+          The position argument cannot be negative.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT bit_get(11, 0);
+           1
+          > SELECT bit_get(11, 2);
+           0
+      
+  since: "3.2.0"
+
+- name: "bitwise_not"
+  scope: "sail_extension"
+  signatures:
+    - "bitwise_not(expr)"
+  usage: "bitwise_not(expr) - Returns the bitwise NOT of expr."
+
+- name: "getbit"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseGet"
+  signatures:
+    - "getbit(expr, pos)"
+  usage: |2-
+    
+        getbit(expr, pos) - Returns the value of the bit (0 or 1) at the specified position.
+          The positions are numbered from right to left, starting at zero.
+          The position argument cannot be negative.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT getbit(11, 0);
+           1
+          > SELECT getbit(11, 2);
+           0
+      
+  since: "3.2.0"
+
+- name: "shiftleft"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ShiftLeft"
+  signatures:
+    - "base shiftleft exp"
+  usage: "base shiftleft exp - Bitwise left shift."
+  examples: |2-
+    
+        Examples:
+          > SELECT shiftleft(2, 1);
+           4
+          > SELECT 2 << 1;
+           4
+      
+  note: |2-
+    
+          `<<` operator is added in Spark 4.0.0 as an alias for `shiftleft`.
+        
+  since: "1.5.0"
+
+- name: "shiftright"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ShiftRight"
+  signatures:
+    - "base shiftright expr"
+  usage: "base shiftright expr - Bitwise (signed) right shift."
+  examples: |2-
+    
+        Examples:
+          > SELECT shiftright(4, 1);
+           2
+          > SELECT 4 >> 1;
+           2
+      
+  note: |2-
+    
+          `>>` operator is added in Spark 4.0.0 as an alias for `shiftright`.
+        
+  since: "1.5.0"
+
+- name: "shiftrightunsigned"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ShiftRightUnsigned"
+  signatures:
+    - "base shiftrightunsigned expr"
+  usage: "base shiftrightunsigned expr - Bitwise unsigned right shift."
+  examples: |2-
+    
+        Examples:
+          > SELECT shiftrightunsigned(4, 1);
+           2
+          > SELECT 4 >>> 1;
+           2
+      
+  note: |2-
+    
+        `>>>` operator is added in Spark 4.0.0 as an alias for `shiftrightunsigned`.
+      
+  since: "1.5.0"
+
+- name: "|"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseOr"
+  signatures:
+    - "expr1 | expr2"
+  usage: "expr1 | expr2 - Returns the result of bitwise OR of `expr1` and `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 3 | 5;
+           7
+      
+  since: "1.4.0"
+
+- name: "~"
+  scope: "sql_builtin"
+  group: "bitwise_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitwiseNot"
+  signatures:
+    - "~ expr"
+  usage: "~ expr - Returns the result of bitwise NOT of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT ~ 0;
+           -1
+      
+  since: "1.4.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/collection.yaml b/crates/sail-plan/data/functions/scalar/collection.yaml
new file mode 100644
index 000000000..d316f3054
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/collection.yaml
@@ -0,0 +1,439 @@
+- name: "aggregate"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayAggregate"
+  signatures:
+    - "aggregate(expr, start, merge, finish)"
+  usage: |2-
+    
+          aggregate(expr, start, merge, finish) - Applies a binary operator to an initial state and all
+          elements in the array, and reduces this to a single state. The final state is converted
+          into the final result by applying a finish function.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x);
+           6
+          > SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10);
+           60
+      
+  since: "2.4.0"
+
+- name: "array_sort"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArraySort"
+  signatures:
+    - "array_sort(expr, func)"
+  usage: |2-
+    array_sort(expr, func) - Sorts the input array. If func is omitted, sort
+        in ascending order. The elements of the input array must be orderable.
+        NaN is greater than any non-NaN elements for double/float type.
+        Null elements will be placed at the end of the returned array.
+        Since 3.0.0 this function also sorts and returns the array based on the
+        given comparator function. The comparator will take two arguments representing
+        two elements of the array.
+        It returns a negative integer, 0, or a positive integer as the first element is less than,
+        equal to, or greater than the second element. If the comparator function returns null,
+        the function will fail and raise an error.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end);
+           [1,5,6]
+          > SELECT array_sort(array('bc', 'ab', 'dc'), (left, right) -> case when left is null and right is null then 0 when left is null then -1 when right is null then 1 when left < right then 1 when left > right then -1 else 0 end);
+           ["dc","bc","ab"]
+          > SELECT array_sort(array('b', 'd', null, 'c', 'a'));
+           ["a","b","c","d",null]
+      
+  since: "2.4.0"
+
+- name: "cardinality"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Size"
+  signatures:
+    - "cardinality(expr)"
+  usage: |2-
+    
+        cardinality(expr) - Returns the size of an array or a map.
+        This function returns -1 for null input only if spark.sql.ansi.enabled is false and
+        spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input.
+        With the default settings, the function returns null for null input.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT cardinality(array('b', 'd', 'c', 'a'));
+           4
+          > SELECT cardinality(map('a', 1, 'b', 2));
+           2
+      
+  since: "2.4.0"
+
+- name: "concat"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Concat"
+  signatures:
+    - "concat(col1, col2, ..., colN)"
+  usage: "concat(col1, col2, ..., colN) - Returns the concatenation of col1, col2, ..., colN."
+  examples: |2-
+    
+        Examples:
+          > SELECT concat('Spark', 'SQL');
+           SparkSQL
+          > SELECT concat(array(1, 2, 3), array(4, 5), array(6));
+           [1,2,3,4,5,6]
+      
+  note: |2-
+    
+        Concat logic for arrays is available since 2.4.0.
+      
+  since: "1.5.0"
+
+- name: "deep_size"
+  scope: "sail_extension"
+  signatures:
+    - "deep_size(expr)"
+  usage: "deep_size(expr) - Returns the number of elements in an array or map."
+
+- name: "element_at"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ElementAt"
+  signatures:
+    - "element_at(array, index)"
+    - "element_at(map, key)"
+  usage: |2-
+    
+        element_at(array, index) - Returns element of array at given (1-based) index. If Index is 0,
+          Spark will throw an error. If index < 0, accesses elements from the last to the first.
+          The function returns NULL if the index exceeds the length of the array and
+          `spark.sql.ansi.enabled` is set to false.
+          If `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException
+          for invalid indices.
+    
+        element_at(map, key) - Returns value for given key. The function returns NULL if the key is not
+           contained in the map.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT element_at(array(1, 2, 3), 2);
+           2
+          > SELECT element_at(map(1, 'a', 2, 'b'), 2);
+           b
+      
+  since: "2.4.0"
+
+- name: "exists"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayExists"
+  signatures:
+    - "exists(expr, pred)"
+  usage: "exists(expr, pred) - Tests whether a predicate holds for one or more elements in the array."
+  examples: |2-
+    
+        Examples:
+          > SELECT exists(array(1, 2, 3), x -> x % 2 == 0);
+           true
+          > SELECT exists(array(1, 2, 3), x -> x % 2 == 10);
+           false
+          > SELECT exists(array(1, null, 3), x -> x % 2 == 0);
+           NULL
+          > SELECT exists(array(0, null, 2, 3, null), x -> x IS NULL);
+           true
+          > SELECT exists(array(1, 2, 3), x -> x IS NULL);
+           false
+      
+  since: "2.4.0"
+
+- name: "filter"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayFilter"
+  signatures:
+    - "filter(expr, func)"
+  usage: "filter(expr, func) - Filters the input array using the given predicate."
+  examples: |2-
+    
+        Examples:
+          > SELECT filter(array(1, 2, 3), x -> x % 2 == 1);
+           [1,3]
+          > SELECT filter(array(0, 2, 3), (x, i) -> x > i);
+           [2,3]
+          > SELECT filter(array(0, null, 2, 3, null), x -> x IS NOT NULL);
+           [0,2,3]
+      
+  note: |2-
+    
+        The inner function may use the index argument since 3.0.0.
+      
+  since: "2.4.0"
+
+- name: "forall"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayForAll"
+  signatures:
+    - "forall(expr, pred)"
+  usage: "forall(expr, pred) - Tests whether a predicate holds for all elements in the array."
+  examples: |2-
+    
+        Examples:
+          > SELECT forall(array(1, 2, 3), x -> x % 2 == 0);
+           false
+          > SELECT forall(array(2, 4, 8), x -> x % 2 == 0);
+           true
+          > SELECT forall(array(1, null, 3), x -> x % 2 == 0);
+           false
+          > SELECT forall(array(2, null, 8), x -> x % 2 == 0);
+           NULL
+      
+  since: "3.0.0"
+
+- name: "map_filter"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapFilter"
+  signatures:
+    - "map_filter(expr, func)"
+  usage: "map_filter(expr, func) - Filters entries in a map using the function."
+  examples: |2-
+    
+        Examples:
+          > SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v);
+           {1:0,3:-1}
+      
+  since: "3.0.0"
+
+- name: "map_zip_with"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapZipWith"
+  signatures:
+    - "map_zip_with(map1, map2, function)"
+  usage: |2-
+    
+          map_zip_with(map1, map2, function) - Merges two given maps into a single map by applying
+          function to the pair of values with the same key. For keys only presented in one map,
+          NULL will be passed as the value for the missing key. If an input map contains duplicated
+          keys, only the first entry of the duplicated key is passed into the lambda function.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2));
+           {1:"ax",2:"by"}
+          > SELECT map_zip_with(map('a', 1, 'b', 2), map('b', 3, 'c', 4), (k, v1, v2) -> coalesce(v1, 0) + coalesce(v2, 0));
+           {"a":1,"b":5,"c":4}
+      
+  since: "3.0.0"
+
+- name: "reduce"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayAggregate"
+  signatures:
+    - "reduce(expr, start, merge, finish)"
+  usage: |2-
+    
+          reduce(expr, start, merge, finish) - Applies a binary operator to an initial state and all
+          elements in the array, and reduces this to a single state. The final state is converted
+          into the final result by applying a finish function.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x);
+           6
+          > SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10);
+           60
+      
+  since: "3.4.0"
+
+- name: "reverse"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Reverse"
+  signatures:
+    - "reverse(expr)"
+  usage: |2-
+    reverse(expr) - Returns a reversed string, a binary value with bytes in reverse order,
+        or an array with reverse order of elements.
+  examples: |2-
+    
+        Examples:
+          > SELECT reverse('Spark SQL');
+           LQS krapS
+          > SELECT reverse(array(2, 1, 4, 3));
+           [3,4,1,2]
+          > SELECT hex(reverse(x'CAFE'));
+           FECA
+      
+  note: |2-
+    
+        Reverse logic for arrays is available since 2.4.0.
+        Reverse logic for binary is available since 4.2.0.
+      
+  since: "1.5.0"
+
+- name: "size"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Size"
+  signatures:
+    - "size(expr)"
+  usage: |2-
+    
+        size(expr) - Returns the size of an array or a map.
+        This function returns -1 for null input only if spark.sql.ansi.enabled is false and
+        spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input.
+        With the default settings, the function returns null for null input.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT size(array('b', 'd', 'c', 'a'));
+           4
+          > SELECT size(map('a', 1, 'b', 2));
+           2
+      
+  since: "1.5.0"
+
+- name: "transform"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ArrayTransform"
+  signatures:
+    - "transform(expr, func)"
+  usage: "transform(expr, func) - Transforms elements in an array using the function."
+  examples: |2-
+    
+        Examples:
+          > SELECT transform(array(1, 2, 3), x -> x + 1);
+           [2,3,4]
+          > SELECT transform(array(1, 2, 3), (x, i) -> x + i);
+           [1,3,5]
+      
+  since: "2.4.0"
+
+- name: "transform_keys"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TransformKeys"
+  signatures:
+    - "transform_keys(expr, func)"
+  usage: "transform_keys(expr, func) - Transforms elements in a map using the function."
+  examples: |2-
+    
+        Examples:
+          > SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);
+           {2:1,3:2,4:3}
+          > SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
+           {2:1,4:2,6:3}
+      
+  since: "3.0.0"
+
+- name: "transform_values"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TransformValues"
+  signatures:
+    - "transform_values(expr, func)"
+  usage: "transform_values(expr, func) - Transforms values in the map using the function."
+  examples: |2-
+    
+        Examples:
+          > SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);
+           {1:2,2:3,3:4}
+          > SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
+           {1:2,2:4,3:6}
+      
+  since: "3.0.0"
+
+- name: "try_element_at"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryElementAt"
+  signatures:
+    - "try_element_at(array, index)"
+    - "try_element_at(map, key)"
+  usage: |2-
+    
+        try_element_at(array, index) - Returns element of array at given (1-based) index. If Index is 0,
+          Spark will throw an error. If index < 0, accesses elements from the last to the first.
+          The function always returns NULL if the index exceeds the length of the array.
+    
+        try_element_at(map, key) - Returns value for given key. The function always returns NULL
+          if the key is not contained in the map.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_element_at(array(1, 2, 3), 2);
+           2
+          > SELECT try_element_at(map(1, 'a', 2, 'b'), 2);
+           b
+      
+  since: "3.3.0"
+
+- name: "zip_with"
+  scope: "sql_builtin"
+  group: "lambda_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ZipWith"
+  signatures:
+    - "zip_with(left, right, func)"
+  usage: "zip_with(left, right, func) - Merges the two given arrays, element-wise, into a single array using function. If one array is shorter, nulls are appended at the end to match the length of the longer array, before applying function."
+  examples: |2-
+    
+        Examples:
+          > SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x));
+           [{"y":"a","x":1},{"y":"b","x":2},{"y":"c","x":3}]
+          > SELECT zip_with(array(1, 2), array(3, 4), (x, y) -> x + y);
+           [4,6]
+          > SELECT zip_with(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y));
+           ["ad","be","cf"]
+      
+  since: "2.4.0"
+
+- name: "||"
+  scope: "sql_builtin"
+  group: "collection_funcs"
+  source: "built-in"
+  signatures:
+    - "expr1 || expr2"
+  usage: "expr1 || expr2 - Returns the concatenation of `expr1` and `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 'Spark' || 'SQL';
+           SparkSQL
+          > SELECT array(1, 2, 3) || array(4, 5) || array(6);
+           [1,2,3,4,5,6]
+  note: |2
+    
+        || for arrays is available since 2.4.0.
+  since: "2.3.0"
diff --git a/crates/sail-plan/data/functions/scalar/conditional.yaml b/crates/sail-plan/data/functions/scalar/conditional.yaml
new file mode 100644
index 000000000..46f5d90cb
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/conditional.yaml
@@ -0,0 +1,201 @@
+- name: "case"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  signatures:
+    - "CASE expr1 WHEN expr2 THEN expr3 [WHEN expr4 THEN expr5]* [ELSE expr6] END"
+  usage: "CASE expr1 WHEN expr2 THEN expr3 [WHEN expr4 THEN expr5]* [ELSE expr6] END - When `expr1` = `expr2`, returns `expr3`; when `expr1` = `expr4`, return `expr5`; else return `expr6`."
+  arguments: |2-
+    
+        Arguments:
+          * expr1 - the expression which is one operand of comparison.
+          * expr2, expr4 - the expressions each of which is the other   operand of comparison.
+          * expr3, expr5, expr6 - the branch value expressions and else value expression  should all be same type or coercible to a common type.
+  examples: |2-
+    
+        Examples:
+          > SELECT CASE col1 WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE '?' END FROM VALUES 1, 2, 3;
+           one
+           two
+           ?
+          > SELECT CASE col1 WHEN 1 THEN 'one' WHEN 2 THEN 'two' END FROM VALUES 1, 2, 3;
+           one
+           two
+           NULL
+  since: "1.0.1"
+
+- name: "coalesce"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Coalesce"
+  signatures:
+    - "coalesce(expr1, expr2, ...)"
+  usage: "coalesce(expr1, expr2, ...) - Returns the first non-null argument if exists. Otherwise, null."
+  examples: |2-
+    
+        Examples:
+          > SELECT coalesce(NULL, 1, NULL);
+           1
+      
+  since: "1.0.0"
+
+- name: "if"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.If"
+  signatures:
+    - "if(expr1, expr2, expr3)"
+  usage: "if(expr1, expr2, expr3) - If `expr1` evaluates to true, then returns `expr2`; otherwise returns `expr3`."
+  examples: |2-
+    
+        Examples:
+          > SELECT if(1 < 2, 'a', 'b');
+           a
+      
+  since: "1.0.0"
+
+- name: "ifnull"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Nvl"
+  signatures:
+    - "ifnull(expr1, expr2)"
+  usage: "ifnull(expr1, expr2) - Returns `expr2` if `expr1` is null, or `expr1` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT ifnull(NULL, array('2'));
+           ["2"]
+      
+  since: "2.0.0"
+
+- name: "nanvl"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.NaNvl"
+  signatures:
+    - "nanvl(expr1, expr2)"
+  usage: "nanvl(expr1, expr2) - Returns `expr1` if it's not NaN, or `expr2` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT nanvl(cast('NaN' as double), 123);
+           123.0
+      
+  since: "1.5.0"
+
+- name: "nullif"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.NullIf"
+  signatures:
+    - "nullif(expr1, expr2)"
+  usage: "nullif(expr1, expr2) - Returns null if `expr1` equals to `expr2`, or `expr1` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT nullif(2, 2);
+           NULL
+      
+  since: "2.0.0"
+
+- name: "nullifzero"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.NullIfZero"
+  signatures:
+    - "nullifzero(expr)"
+  usage: "nullifzero(expr) - Returns null if `expr` is equal to zero, or `expr` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT nullifzero(0);
+           NULL
+          > SELECT nullifzero(2);
+           2
+      
+  since: "4.0.0"
+
+- name: "nvl"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Nvl"
+  signatures:
+    - "nvl(expr1, expr2)"
+  usage: "nvl(expr1, expr2) - Returns `expr2` if `expr1` is null, or `expr1` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT nvl(NULL, array('2'));
+           ["2"]
+      
+  since: "2.0.0"
+
+- name: "nvl2"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Nvl2"
+  signatures:
+    - "nvl2(expr1, expr2, expr3)"
+  usage: "nvl2(expr1, expr2, expr3) - Returns `expr2` if `expr1` is not null, or `expr3` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT nvl2(NULL, 2, 1);
+           1
+      
+  since: "2.0.0"
+
+- name: "when"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CaseWhen"
+  signatures:
+    - "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END"
+  usage: "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END - When `expr1` = true, returns `expr2`; else when `expr3` = true, returns `expr4`; else returns `expr5`."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr3 - the branch condition expressions should all be boolean type.
+          * expr2, expr4, expr5 - the branch value expressions and else value expression should all be
+              same type or coercible to a common type.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;
+           1.0
+          > SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;
+           2.0
+          > SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 < 0 THEN 2.0 END;
+           NULL
+      
+  since: "1.0.1"
+
+- name: "zeroifnull"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ZeroIfNull"
+  signatures:
+    - "zeroifnull(expr)"
+  usage: "zeroifnull(expr) - Returns zero if `expr` is equal to null, or `expr` otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT zeroifnull(NULL);
+           0
+          > SELECT zeroifnull(2);
+           2
+      
+  since: "4.0.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/conversion.yaml b/crates/sail-plan/data/functions/scalar/conversion.yaml
new file mode 100644
index 000000000..ad162db61
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/conversion.yaml
@@ -0,0 +1,148 @@
+- name: "bigint"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "bigint(expr)"
+  usage: "bigint(expr) - Casts the value `expr` to the target data type `bigint`."
+  since: "2.0.1"
+
+- name: "binary"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "binary(expr)"
+  usage: "binary(expr) - Casts the value `expr` to the target data type `binary`."
+  since: "2.0.1"
+
+- name: "boolean"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "boolean(expr)"
+  usage: "boolean(expr) - Casts the value `expr` to the target data type `boolean`."
+  since: "2.0.1"
+
+- name: "cast"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "cast(expr AS type)"
+  usage: "cast(expr AS type) - Casts the value `expr` to the target data type `type`. `expr` :: `type` alternative casting syntax is also supported."
+  examples: |2-
+    
+        Examples:
+          > SELECT cast('10' as int);
+           10
+          > SELECT '10' :: int;
+           10
+      
+  since: "1.0.0"
+
+- name: "date"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "date(expr)"
+  usage: "date(expr) - Casts the value `expr` to the target data type `date`."
+  since: "2.0.1"
+
+- name: "decimal"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "decimal(expr)"
+  usage: "decimal(expr) - Casts the value `expr` to the target data type `decimal`."
+  since: "2.0.1"
+
+- name: "double"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "double(expr)"
+  usage: "double(expr) - Casts the value `expr` to the target data type `double`."
+  since: "2.0.1"
+
+- name: "float"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "float(expr)"
+  usage: "float(expr) - Casts the value `expr` to the target data type `float`."
+  since: "2.0.1"
+
+- name: "int"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "int(expr)"
+  usage: "int(expr) - Casts the value `expr` to the target data type `int`."
+  since: "2.0.1"
+
+- name: "smallint"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "smallint(expr)"
+  usage: "smallint(expr) - Casts the value `expr` to the target data type `smallint`."
+  since: "2.0.1"
+
+- name: "string"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "string(expr)"
+  usage: "string(expr) - Casts the value `expr` to the target data type `string`."
+  since: "2.0.1"
+
+- name: "time"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "time(expr)"
+  usage: "time(expr) - Casts the value `expr` to the target data type `time`."
+  since: "2.0.1"
+
+- name: "timestamp"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "timestamp(expr)"
+  usage: "timestamp(expr) - Casts the value `expr` to the target data type `timestamp`."
+  since: "2.0.1"
+
+- name: "tinyint"
+  scope: "sql_builtin"
+  group: "conversion_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cast"
+  signatures:
+    - "tinyint(expr)"
+  usage: "tinyint(expr) - Casts the value `expr` to the target data type `tinyint`."
+  since: "2.0.1"
+
diff --git a/crates/sail-plan/data/functions/scalar/csv.yaml b/crates/sail-plan/data/functions/scalar/csv.yaml
new file mode 100644
index 000000000..2e337093c
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/csv.yaml
@@ -0,0 +1,52 @@
+- name: "from_csv"
+  scope: "sql_builtin"
+  group: "csv_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CsvToStructs"
+  signatures:
+    - "from_csv(csvStr, schema[, options])"
+  usage: "from_csv(csvStr, schema[, options]) - Returns a struct value with the given `csvStr` and `schema`."
+  examples: |2-
+    
+        Examples:
+          > SELECT from_csv('1, 0.8', 'a INT, b DOUBLE');
+           {"a":1,"b":0.8}
+          > SELECT from_csv('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
+           {"time":2015-08-26 00:00:00}
+      
+  since: "3.0.0"
+
+- name: "schema_of_csv"
+  scope: "sql_builtin"
+  group: "csv_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SchemaOfCsv"
+  signatures:
+    - "schema_of_csv(csv[, options])"
+  usage: "schema_of_csv(csv[, options]) - Returns schema in the DDL format of CSV string."
+  examples: |2-
+    
+        Examples:
+          > SELECT schema_of_csv('1,abc');
+           STRUCT<_c0: INT, _c1: STRING>
+      
+  since: "3.0.0"
+
+- name: "to_csv"
+  scope: "sql_builtin"
+  group: "csv_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StructsToCsv"
+  signatures:
+    - "to_csv(expr[, options])"
+  usage: "to_csv(expr[, options]) - Returns a CSV string with a given struct value"
+  examples: |2-
+    
+        Examples:
+          > SELECT to_csv(named_struct('a', 1, 'b', 2));
+           1,2
+          > SELECT to_csv(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));
+           26/08/2015
+      
+  since: "3.0.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/datetime.yaml b/crates/sail-plan/data/functions/scalar/datetime.yaml
new file mode 100644
index 000000000..c5a3d6be1
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/datetime.yaml
@@ -0,0 +1,2178 @@
+- name: "add_days"
+  scope: "sail_extension"
+  signatures:
+    - "add_days(start_date, num_days)"
+  usage: "add_days(start_date, num_days) - Returns the date that is num_days after start_date."
+
+- name: "add_months"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.AddMonths"
+  signatures:
+    - "add_months(start_date, num_months)"
+  usage: "add_months(start_date, num_months) - Returns the date that is `num_months` after `start_date`."
+  examples: |2-
+    
+        Examples:
+          > SELECT add_months('2016-08-31', 1);
+           2016-09-30
+      
+  since: "1.5.0"
+
+- name: "add_years"
+  scope: "sail_extension"
+  signatures:
+    - "add_years(start_date, num_years)"
+  usage: "add_years(start_date, num_years) - Returns the date that is num_years after start_date."
+
+- name: "convert_timezone"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ConvertTimezone"
+  signatures:
+    - "convert_timezone([sourceTz, ]targetTz, sourceTs)"
+  usage: "convert_timezone([sourceTz, ]targetTz, sourceTs) - Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to `targetTz`. "
+  arguments: |2-
+    
+        Arguments:
+          * sourceTz - the time zone for the input timestamp.
+                       If it is missed, the current session time zone is used as the source time zone.
+          * targetTz - the time zone to which the input timestamp should be converted
+          * sourceTs - a timestamp without time zone
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00');
+           2021-12-05 15:00:00
+          > SELECT convert_timezone('Europe/Brussels', timestamp_ntz'2021-12-05 15:00:00');
+           2021-12-06 00:00:00
+      
+  since: "3.4.0"
+
+- name: "curdate"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurDateExpressionBuilder"
+  signatures:
+    - "curdate()"
+  usage: |2-
+    
+        curdate() - Returns the current date at the start of query evaluation. All calls of curdate within the same query return the same value.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT curdate();
+           2022-09-06
+      
+  since: "3.4.0"
+
+- name: "current_date"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentDate"
+  signatures:
+    - "current_date()"
+    - "current_date"
+  usage: |2-
+    
+        current_date() - Returns the current date at the start of query evaluation. All calls of current_date within the same query return the same value.
+    
+        current_date - Returns the current date at the start of query evaluation.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT current_date();
+           2020-04-25
+          > SELECT current_date;
+           2020-04-25
+      
+  note: |2-
+    
+        The syntax without braces has been supported since 2.0.1.
+      
+  since: "1.5.0"
+
+- name: "current_time"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentTime"
+  signatures:
+    - "current_time([precision])"
+    - "current_time"
+  usage: |2-
+    
+        current_time([precision]) - Returns the current time at the start of query evaluation.
+        All calls of current_time within the same query return the same value.
+    
+        current_time - Returns the current time at the start of query evaluation.
+      
+  arguments: |2-
+    
+        Arguments:
+          * precision - An optional integer literal in the range [0..6], indicating how many
+                        fractional digits of seconds to include. If omitted, the default is 6.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT current_time();
+           15:49:11.914120
+          > SELECT current_time;
+           15:49:11.914120
+          > SELECT current_time(0);
+           15:49:11
+          > SELECT current_time(3);
+           15:49:11.914
+          > SELECT current_time(1+1);
+           15:49:11.91
+      
+  since: "4.1.0"
+
+- name: "current_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp"
+  signatures:
+    - "current_timestamp()"
+    - "current_timestamp"
+  usage: |2-
+    
+        current_timestamp() - Returns the current timestamp at the start of query evaluation. All calls of current_timestamp within the same query return the same value.
+    
+        current_timestamp - Returns the current timestamp at the start of query evaluation.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT current_timestamp();
+           2020-04-25 15:49:11.914
+          > SELECT current_timestamp;
+           2020-04-25 15:49:11.914
+      
+  note: |2-
+    
+        The syntax without braces has been supported since 2.0.1.
+      
+  since: "1.5.0"
+
+- name: "current_timezone"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentTimeZone"
+  signatures:
+    - "current_timezone()"
+  usage: "current_timezone() - Returns the current session local timezone."
+  examples: |2-
+    
+        Examples:
+          > SELECT current_timezone();
+           Asia/Shanghai
+      
+  since: "3.1.0"
+
+- name: "date_add"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateAdd"
+  signatures:
+    - "date_add(start_date, num_days)"
+  usage: "date_add(start_date, num_days) - Returns the date that is `num_days` after `start_date`."
+  examples: |2-
+    
+        Examples:
+          > SELECT date_add('2016-07-30', 1);
+           2016-07-31
+      
+  since: "1.5.0"
+
+- name: "date_diff"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateDiff"
+  signatures:
+    - "date_diff(endDate, startDate)"
+  usage: "date_diff(endDate, startDate) - Returns the number of days from `startDate` to `endDate`."
+  examples: |2-
+    
+        Examples:
+          > SELECT date_diff('2009-07-31', '2009-07-30');
+           1
+    
+          > SELECT date_diff('2009-07-30', '2009-07-31');
+           -1
+      
+  since: "3.4.0"
+
+- name: "date_format"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateFormatClass"
+  signatures:
+    - "date_format(timestamp, fmt)"
+  usage: "date_format(timestamp, fmt) - Converts `timestamp` to a value of string in the format specified by the date format `fmt`."
+  arguments: |2-
+    
+        Arguments:
+          * timestamp - A date/timestamp or string to be converted to the given format.
+          * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date
+                  and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT date_format('2016-04-08', 'y');
+           2016
+      
+  since: "1.5.0"
+
+- name: "date_from_unix_date"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateFromUnixDate"
+  signatures:
+    - "date_from_unix_date(days)"
+  usage: "date_from_unix_date(days) - Create date from the number of days since 1970-01-01."
+  examples: |2-
+    
+        Examples:
+          > SELECT date_from_unix_date(1);
+           1970-01-02
+      
+  since: "3.1.0"
+
+- name: "date_part"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DatePartExpressionBuilder"
+  signatures:
+    - "date_part(field, source)"
+  usage: "date_part(field, source) - Extracts a part of the date/timestamp or interval source."
+  arguments: |2-
+    
+        Arguments:
+          * field - selects which part of the source should be extracted, and supported string values are as same as the fields of the equivalent function `EXTRACT`.
+          * source - a date/timestamp or interval column from where `field` should be extracted
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456');
+           2019
+          > SELECT date_part('week', timestamp'2019-08-12 01:00:00.123456');
+           33
+          > SELECT date_part('doy', DATE'2019-08-12');
+           224
+          > SELECT date_part('SECONDS', timestamp'2019-10-01 00:00:01.000001');
+           1.000001
+          > SELECT date_part('days', interval 5 days 3 hours 7 minutes);
+           5
+          > SELECT date_part('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
+           30.001001
+          > SELECT date_part('MONTH', INTERVAL '2021-11' YEAR TO MONTH);
+           11
+          > SELECT date_part('MINUTE', INTERVAL '123 23:55:59.002001' DAY TO SECOND);
+           55
+      
+  note: |2-
+    
+        The date_part function is equivalent to the SQL-standard function `EXTRACT(field FROM source)`
+      
+  since: "3.0.0"
+
+- name: "date_sub"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateSub"
+  signatures:
+    - "date_sub(start_date, num_days)"
+  usage: "date_sub(start_date, num_days) - Returns the date that is `num_days` before `start_date`."
+  examples: |2-
+    
+        Examples:
+          > SELECT date_sub('2016-07-30', 1);
+           2016-07-29
+      
+  since: "1.5.0"
+
+- name: "date_trunc"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TruncTimestamp"
+  signatures:
+    - "date_trunc(fmt, ts)"
+  usage: |2-
+    
+        date_trunc(fmt, ts) - Returns timestamp `ts` truncated to the unit specified by the format model `fmt`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * fmt - the format representing the unit to be truncated to
+              - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `ts` falls in, the time part will be zero out
+              - "QUARTER" - truncate to the first date of the quarter that the `ts` falls in, the time part will be zero out
+              - "MONTH", "MM", "MON" - truncate to the first date of the month that the `ts` falls in, the time part will be zero out
+              - "WEEK" - truncate to the Monday of the week that the `ts` falls in, the time part will be zero out
+              - "DAY", "DD" - zero out the time part
+              - "HOUR" - zero out the minute and second with fraction part
+              - "MINUTE"- zero out the second with fraction part
+              - "SECOND" -  zero out the second fraction part
+              - "MILLISECOND" - zero out the microseconds
+              - "MICROSECOND" - everything remains
+          * ts - datetime value or valid timestamp string
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359');
+           2015-01-01 00:00:00
+          > SELECT date_trunc('MM', '2015-03-05T09:32:05.359');
+           2015-03-01 00:00:00
+          > SELECT date_trunc('DD', '2015-03-05T09:32:05.359');
+           2015-03-05 00:00:00
+          > SELECT date_trunc('HOUR', '2015-03-05T09:32:05.359');
+           2015-03-05 09:00:00
+          > SELECT date_trunc('MILLISECOND', '2015-03-05T09:32:05.123456');
+           2015-03-05 09:32:05.123
+      
+  since: "2.3.0"
+
+- name: "dateadd"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateAdd"
+  signatures:
+    - "dateadd(start_date, num_days)"
+  usage: "dateadd(start_date, num_days) - Returns the date that is `num_days` after `start_date`."
+  examples: |2-
+    
+        Examples:
+          > SELECT dateadd('2016-07-30', 1);
+           2016-07-31
+      
+  since: "3.4.0"
+
+- name: "datediff"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DateDiff"
+  signatures:
+    - "datediff(endDate, startDate)"
+  usage: "datediff(endDate, startDate) - Returns the number of days from `startDate` to `endDate`."
+  examples: |2-
+    
+        Examples:
+          > SELECT datediff('2009-07-31', '2009-07-30');
+           1
+    
+          > SELECT datediff('2009-07-30', '2009-07-31');
+           -1
+      
+  since: "1.5.0"
+
+- name: "datepart"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DatePartExpressionBuilder"
+  signatures:
+    - "datepart(field, source)"
+  usage: "datepart(field, source) - Extracts a part of the date/timestamp or interval source."
+  arguments: |2-
+    
+        Arguments:
+          * field - selects which part of the source should be extracted, and supported string values are as same as the fields of the equivalent function `EXTRACT`.
+          * source - a date/timestamp or interval column from where `field` should be extracted
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456');
+           2019
+          > SELECT datepart('week', timestamp'2019-08-12 01:00:00.123456');
+           33
+          > SELECT datepart('doy', DATE'2019-08-12');
+           224
+          > SELECT datepart('SECONDS', timestamp'2019-10-01 00:00:01.000001');
+           1.000001
+          > SELECT datepart('days', interval 5 days 3 hours 7 minutes);
+           5
+          > SELECT datepart('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
+           30.001001
+          > SELECT datepart('MONTH', INTERVAL '2021-11' YEAR TO MONTH);
+           11
+          > SELECT datepart('MINUTE', INTERVAL '123 23:55:59.002001' DAY TO SECOND);
+           55
+      
+  note: |2-
+    
+        The datepart function is equivalent to the SQL-standard function `EXTRACT(field FROM source)`
+      
+  since: "3.4.0"
+
+- name: "day"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DayOfMonth"
+  signatures:
+    - "day(date)"
+  usage: "day(date) - Returns the day of month of the date/timestamp."
+  examples: |2-
+    
+        Examples:
+          > SELECT day('2009-07-30');
+           30
+      
+  since: "1.5.0"
+
+- name: "dayname"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DayName"
+  signatures:
+    - "dayname(date)"
+  usage: "dayname(date) - Returns the three-letter abbreviated day name from the given date."
+  examples: |2-
+    
+        Examples:
+          > SELECT dayname(DATE('2008-02-20'));
+           Wed
+      
+  since: "4.0.0"
+
+- name: "dayofmonth"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DayOfMonth"
+  signatures:
+    - "dayofmonth(date)"
+  usage: "dayofmonth(date) - Returns the day of month of the date/timestamp."
+  examples: |2-
+    
+        Examples:
+          > SELECT dayofmonth('2009-07-30');
+           30
+      
+  since: "1.5.0"
+
+- name: "dayofweek"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DayOfWeek"
+  signatures:
+    - "dayofweek(date)"
+  usage: "dayofweek(date) - Returns the day of the week for date/timestamp (1 = Sunday, 2 = Monday, ..., 7 = Saturday)."
+  examples: |2-
+    
+        Examples:
+          > SELECT dayofweek('2009-07-30');
+           5
+      
+  since: "2.3.0"
+
+- name: "dayofyear"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DayOfYear"
+  signatures:
+    - "dayofyear(date)"
+  usage: "dayofyear(date) - Returns the day of year of the date/timestamp."
+  examples: |2-
+    
+        Examples:
+          > SELECT dayofyear('2016-04-09');
+           100
+      
+  since: "1.5.0"
+
+- name: "extract"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Extract"
+  signatures:
+    - "extract(field FROM source)"
+  usage: "extract(field FROM source) - Extracts a part of the date or timestamp or time or interval source."
+  arguments: |2-
+    
+        Arguments:
+          * field - selects which part of the source should be extracted
+              - Supported string values of `field` for dates and timestamps are(case insensitive):
+                  - "YEAR", ("Y", "YEARS", "YR", "YRS") - the year field
+                  - "YEAROFWEEK" - the ISO 8601 week-numbering year that the datetime falls in. For example, 2005-01-02 is part of the 53rd week of year 2004, so the result is 2004
+                  - "QUARTER", ("QTR") - the quarter (1 - 4) of the year that the datetime falls in
+                  - "MONTH", ("MON", "MONS", "MONTHS") - the month field (1 - 12)
+                  - "WEEK", ("W", "WEEKS") - the number of the ISO 8601 week-of-week-based-year. A week is considered to start on a Monday and week 1 is the first week with >3 days. In the ISO week-numbering system, it is possible for early-January dates to be part of the 52nd or 53rd week of the previous year, and for late-December dates to be part of the first week of the next year. For example, 2005-01-02 is part of the 53rd week of year 2004, while 2012-12-31 is part of the first week of 2013
+                  - "DAY", ("D", "DAYS") - the day of the month field (1 - 31)
+                  - "DAYOFWEEK",("DOW") - the day of the week for datetime as Sunday(1) to Saturday(7)
+                  - "DAYOFWEEK_ISO",("DOW_ISO") - ISO 8601 based day of the week for datetime as Monday(1) to Sunday(7)
+                  - "DOY" - the day of the year (1 - 365/366)
+                  - "HOUR", ("H", "HOURS", "HR", "HRS") - The hour field (0 - 23)
+                  - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - the minutes field (0 - 59)
+                  - "SECOND", ("S", "SEC", "SECONDS", "SECS") - the seconds field, including fractional parts
+              - Supported string values of `field` for interval(which consists of `months`, `days`, `microseconds`) are(case insensitive):
+                  - "YEAR", ("Y", "YEARS", "YR", "YRS") - the total `months` / 12
+                  - "MONTH", ("MON", "MONS", "MONTHS") - the total `months` % 12
+                  - "DAY", ("D", "DAYS") - the `days` part of interval
+                  - "HOUR", ("H", "HOURS", "HR", "HRS") - how many hours the `microseconds` contains
+                  - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - how many minutes left after taking hours from `microseconds`
+                  - "SECOND", ("S", "SEC", "SECONDS", "SECS") - how many second with fractions left after taking hours and minutes from `microseconds`
+              - Supported string values of `field` for time (which consists of `hour`, `minute`, `second`) are(case insensitive):
+                  - "HOUR", ("H", "HOURS", "HR", "HRS") - The hour field (0 - 23)
+                  - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - the minutes field (0 - 59)
+                  - "SECOND", ("S", "SEC", "SECONDS", "SECS") - the seconds field, including fractional parts up to micro second precision. Returns a DECIMAL(8, 6) precision value.
+          * source - a date or timestamp or time or interval column from where `field` should be extracted
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456');
+           2019
+          > SELECT extract(week FROM timestamp'2019-08-12 01:00:00.123456');
+           33
+          > SELECT extract(doy FROM DATE'2019-08-12');
+           224
+          > SELECT extract(SECONDS FROM timestamp'2019-10-01 00:00:01.000001');
+           1.000001
+          > SELECT extract(days FROM interval 5 days 3 hours 7 minutes);
+           5
+          > SELECT extract(seconds FROM interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
+           30.001001
+          > SELECT extract(MONTH FROM INTERVAL '2021-11' YEAR TO MONTH);
+           11
+          > SELECT extract(MINUTE FROM INTERVAL '123 23:55:59.002001' DAY TO SECOND);
+           55
+          > SELECT extract(HOUR FROM time '09:08:01.000001');
+           9
+          > SELECT extract(MINUTE FROM time '09:08:01.000001');
+           8
+          > SELECT extract(SECOND FROM time '09:08:01.000001');
+           1.000001
+      
+  note: |2-
+    
+        The extract function is equivalent to `date_part(field, source)`.
+      
+  since: "3.0.0"
+
+- name: "from_unixtime"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FromUnixTime"
+  signatures:
+    - "from_unixtime(unix_time[, fmt])"
+  usage: "from_unixtime(unix_time[, fmt]) - Returns `unix_time` in the specified `fmt`."
+  arguments: |2-
+    
+        Arguments:
+          * unix_time - UNIX Timestamp to be converted to the provided format.
+          * fmt - Date/time format pattern to follow. See Datetime Patterns
+                  for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss');
+           1969-12-31 16:00:00
+    
+          > SELECT from_unixtime(0);
+           1969-12-31 16:00:00
+      
+  since: "1.5.0"
+
+- name: "from_utc_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FromUTCTimestamp"
+  signatures:
+    - "from_utc_timestamp(timestamp, timezone)"
+  usage: "from_utc_timestamp(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'."
+  examples: |2-
+    
+        Examples:
+          > SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul');
+           2016-08-31 09:00:00
+      
+  since: "1.5.0"
+
+- name: "hour"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.HourExpressionBuilder"
+  signatures:
+    - "hour(expr)"
+  usage: |2-
+    
+        hour(expr) - Returns the hour component of the given expression.
+    
+        If `expr` is a TIMESTAMP or a string that can be cast to timestamp,
+        it returns the hour of that timestamp.
+        If `expr` is a TIME type (since 4.1.0), it returns the hour of the time-of-day.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT hour('2018-02-14 12:58:59');
+           12
+          > SELECT hour(TIME'13:59:59.999999');
+           13
+      
+  since: "1.5.0"
+
+- name: "last_day"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.LastDay"
+  signatures:
+    - "last_day(date)"
+  usage: "last_day(date) - Returns the last day of the month which the date belongs to."
+  examples: |2-
+    
+        Examples:
+          > SELECT last_day('2009-01-12');
+           2009-01-31
+      
+  since: "1.5.0"
+
+- name: "localtimestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.LocalTimestamp"
+  signatures:
+    - "localtimestamp()"
+    - "localtimestamp"
+  usage: |2-
+    
+        localtimestamp() - Returns the current timestamp without time zone at the start of query evaluation. All calls of localtimestamp within the same query return the same value.
+    
+        localtimestamp - Returns the current local date-time at the session time zone at the start of query evaluation.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT localtimestamp();
+           2020-04-25 15:49:11.914
+      
+  since: "3.4.0"
+
+- name: "make_date"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeDate"
+  signatures:
+    - "make_date(year, month, day)"
+  usage: "make_date(year, month, day) - Create date from year, month and day fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error instead."
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_date(2013, 7, 15);
+           2013-07-15
+          > SELECT make_date(2019, 7, NULL);
+           NULL
+      
+  since: "3.0.0"
+
+- name: "make_dt_interval"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeDTInterval"
+  signatures:
+    - "make_dt_interval([days[, hours[, mins[, secs]]]])"
+  usage: "make_dt_interval([days[, hours[, mins[, secs]]]]) - Make DayTimeIntervalType duration from days, hours, mins and secs."
+  arguments: |2-
+    
+        Arguments:
+          * days - the number of days, positive or negative
+          * hours - the number of hours, positive or negative
+          * mins - the number of minutes, positive or negative
+          * secs - the number of seconds with the fractional part in microsecond precision.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_dt_interval(1, 12, 30, 01.001001);
+           1 12:30:01.001001000
+          > SELECT make_dt_interval(2);
+           2 00:00:00.000000000
+          > SELECT make_dt_interval(100, null, 3);
+           NULL
+      
+  since: "3.2.0"
+
+- name: "make_interval"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeInterval"
+  signatures:
+    - "make_interval([years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]])"
+  usage: "make_interval([years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]]) - Make interval from years, months, weeks, days, hours, mins and secs."
+  arguments: |2-
+    
+        Arguments:
+          * years - the number of years, positive or negative
+          * months - the number of months, positive or negative
+          * weeks - the number of weeks, positive or negative
+          * days - the number of days, positive or negative
+          * hours - the number of hours, positive or negative
+          * mins - the number of minutes, positive or negative
+          * secs - the number of seconds with the fractional part in microsecond precision.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001);
+           100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds
+          > SELECT make_interval(100, null, 3);
+           NULL
+          > SELECT make_interval(0, 1, 0, 1, 0, 0, 100.000001);
+           1 months 1 days 1 minutes 40.000001 seconds
+      
+  since: "3.0.0"
+
+- name: "make_time"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeTime"
+  signatures:
+    - "make_time(hour, minute, second)"
+  usage: "make_time(hour, minute, second) - Create time from hour, minute and second fields. For invalid inputs it will throw an error."
+  arguments: |2-
+    
+        Arguments:
+          * hour - the hour to represent, from 0 to 23
+          * minute - the minute to represent, from 0 to 59
+          * second - the second to represent, from 0 to 59.999999
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_time(6, 30, 45.887);
+           06:30:45.887
+          > SELECT make_time(NULL, 30, 0);
+           NULL
+      
+  since: "4.1.0"
+
+- name: "make_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeTimestampExpressionBuilder"
+  signatures:
+    - "make_timestamp(year, month, day, hour, min, sec[, timezone])"
+    - "make_timestamp(date[, time[, timezone]])"
+  usage: |2-
+    
+        make_timestamp(year, month, day, hour, min, sec[, timezone]) - Create the current timestamp with local time zone from year, month, day, hour, min, sec and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error instead.
+    
+        make_timestamp(date[, time[, timezone]]) - Create timestamp from date and time fields.
+        
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+          * hour - the hour-of-day to represent, from 0 to 23
+          * min - the minute-of-hour to represent, from 0 to 59
+          * sec - the second-of-minute and its micro-fraction to represent, from
+                  0 to 60. If the sec argument equals to 60, the seconds field is set
+                  to 0 and 1 minute is added to the final timestamp.
+          * date - a date expression
+          * time - a time expression (optional). Default is 00:00:00.
+          * timezone - the time zone identifier (optional). For example, CET, UTC and etc.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887);
+           2014-12-28 06:30:45.887
+          > SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887, 'CET');
+           2014-12-27 21:30:45.887
+          > SELECT make_timestamp(DATE'2014-12-28');
+           2014-12-28 00:00:00
+          > SELECT make_timestamp(DATE'2014-12-28', TIME'6:30:45.887');
+           2014-12-28 06:30:45.887
+          > SELECT make_timestamp(DATE'2014-12-28', TIME'6:30:45.887', 'CET');
+           2014-12-27 21:30:45.887
+      
+  since: "3.0.0"
+
+- name: "make_timestamp_ltz"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeTimestampLTZExpressionBuilder"
+  signatures:
+    - "make_timestamp_ltz(year, month, day, hour, min, sec[, timezone])"
+    - "make_timestamp_ltz(date, time[, timezone])"
+  usage: |2-
+    
+        make_timestamp_ltz(year, month, day, hour, min, sec[, timezone]) - Create the current timestamp with local time zone from year, month, day, hour, min, sec and (optional) timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error instead.
+    
+        make_timestamp_ltz(date, time[, timezone]) - Create a local date-time from date, time and (optional) timezone fields.
+        
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+          * hour - the hour-of-day to represent, from 0 to 23
+          * min - the minute-of-hour to represent, from 0 to 59
+          * sec - the second-of-minute and its micro-fraction to represent, from
+                  0 to 60. If the sec argument equals to 60, the seconds field is set
+                  to 0 and 1 minute is added to the final timestamp.
+          * timezone - the time zone identifier. For example, CET, UTC and etc.
+          * date - a date to represent, from 0001-01-01 to 9999-12-31
+          * time - a local time to represent, from 00:00:00 to 23:59:59.999999
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887);
+           2014-12-28 06:30:45.887
+          > SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887, 'CET');
+           2014-12-27 21:30:45.887
+          > SELECT make_timestamp_ltz(2019, 6, 30, 23, 59, 60);
+           2019-07-01 00:00:00
+          > SELECT make_timestamp_ltz(null, 7, 22, 15, 30, 0);
+           NULL
+          > SELECT make_timestamp_ltz(DATE'2014-12-28', TIME'6:30:45.887');
+           2014-12-28 06:30:45.887
+          > SELECT make_timestamp_ltz(DATE'2014-12-28', TIME'6:30:45.887', 'CET');
+           2014-12-27 21:30:45.887
+      
+  since: "3.4.0"
+
+- name: "make_timestamp_ntz"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeTimestampNTZExpressionBuilder"
+  signatures:
+    - "make_timestamp_ntz(year, month, day, hour, min, sec)"
+    - "make_timestamp_ntz(date, time)"
+  usage: |2-
+    
+        make_timestamp_ntz(year, month, day, hour, min, sec) - Create local date-time from year, month, day, hour, min, sec fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error instead.
+    
+        make_timestamp_ntz(date, time) - Create a local date-time from date and time fields.
+        
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+          * hour - the hour-of-day to represent, from 0 to 23
+          * min - the minute-of-hour to represent, from 0 to 59
+          * sec - the second-of-minute and its micro-fraction to represent, from
+                  0 to 60. If the sec argument equals to 60, the seconds field is set
+                  to 0 and 1 minute is added to the final timestamp.
+          * date - a date to represent, from 0001-01-01 to 9999-12-31
+          * time - a local time to represent, from 00:00:00 to 23:59:59.999999
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887);
+           2014-12-28 06:30:45.887
+          > SELECT make_timestamp_ntz(2019, 6, 30, 23, 59, 60);
+           2019-07-01 00:00:00
+          > SELECT make_timestamp_ntz(null, 7, 22, 15, 30, 0);
+           NULL
+          > SELECT make_timestamp_ntz(DATE'2014-12-28', TIME'6:30:45.887');
+           2014-12-28 06:30:45.887
+      
+  since: "3.4.0"
+
+- name: "make_ym_interval"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeYMInterval"
+  signatures:
+    - "make_ym_interval([years[, months]])"
+  usage: "make_ym_interval([years[, months]]) - Make year-month interval from years, months."
+  arguments: |2-
+    
+        Arguments:
+          * years - the number of years, positive or negative
+          * months - the number of months, positive or negative
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_ym_interval(1, 2);
+           1-2
+          > SELECT make_ym_interval(1, 0);
+           1-0
+          > SELECT make_ym_interval(-1, 1);
+           -0-11
+          > SELECT make_ym_interval(2);
+           2-0
+      
+  since: "3.2.0"
+
+- name: "minute"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MinuteExpressionBuilder"
+  signatures:
+    - "minute(expr)"
+  usage: |2-
+    
+        minute(expr) - Returns the minute component of the given expression.
+    
+        If `expr` is a TIMESTAMP or a string that can be cast to timestamp,
+        it returns the minute of that timestamp.
+        If `expr` is a TIME type (since 4.1.0), it returns the minute of the time-of-day.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT minute('2009-07-30 12:58:59');
+           58
+          > SELECT minute(TIME'23:59:59.999999');
+           59
+      
+  since: "1.5.0"
+
+- name: "month"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Month"
+  signatures:
+    - "month(date)"
+  usage: "month(date) - Returns the month component of the date/timestamp."
+  examples: |2-
+    
+        Examples:
+          > SELECT month('2016-07-30');
+           7
+      
+  since: "1.5.0"
+
+- name: "monthname"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MonthName"
+  signatures:
+    - "monthname(date)"
+  usage: "monthname(date) - Returns the three-letter abbreviated month name from the given date."
+  examples: |2-
+    
+        Examples:
+          > SELECT monthname('2008-02-20');
+           Feb
+      
+  since: "4.0.0"
+
+- name: "months_between"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MonthsBetween"
+  signatures:
+    - "months_between(timestamp1, timestamp2[, roundOff])"
+  usage: |2-
+    
+        months_between(timestamp1, timestamp2[, roundOff]) - If `timestamp1` is later than `timestamp2`, then the result
+          is positive. If `timestamp1` and `timestamp2` are on the same day of month, or both
+          are the last day of month, time of day will be ignored. Otherwise, the difference is
+          calculated based on 31 days per month, and rounded to 8 digits unless roundOff=false.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT months_between('1997-02-28 10:30:00', '1996-10-30');
+           3.94959677
+          > SELECT months_between('1997-02-28 10:30:00', '1996-10-30', false);
+           3.9495967741935485
+      
+  since: "1.5.0"
+
+- name: "next_day"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.NextDay"
+  signatures:
+    - "next_day(start_date, day_of_week)"
+  usage: |2-
+    next_day(start_date, day_of_week) - Returns the first date which is later than `start_date` and named as indicated.
+          The function returns NULL if at least one of the input parameters is NULL.
+          When both of the input parameters are not NULL and day_of_week is an invalid input,
+          the function throws SparkIllegalArgumentException if `spark.sql.ansi.enabled` is set to true, otherwise NULL.
+          
+  examples: |2-
+    
+        Examples:
+          > SELECT next_day('2015-01-14', 'TU');
+           2015-01-20
+      
+  since: "1.5.0"
+
+- name: "now"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Now"
+  signatures:
+    - "now()"
+  usage: "now() - Returns the current timestamp at the start of query evaluation."
+  examples: |2-
+    
+        Examples:
+          > SELECT now();
+           2020-04-25 15:49:11.914
+      
+  since: "1.6.0"
+
+- name: "quarter"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Quarter"
+  signatures:
+    - "quarter(date)"
+  usage: "quarter(date) - Returns the quarter of the year for date, in the range 1 to 4."
+  examples: |2-
+    
+        Examples:
+          > SELECT quarter('2016-08-31');
+           3
+      
+  since: "1.5.0"
+
+- name: "second"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SecondExpressionBuilder"
+  signatures:
+    - "second(expr)"
+  usage: |2-
+    
+        second(expr) - Returns the second component of the given expression.
+    
+        If `expr` is a TIMESTAMP or a string that can be cast to timestamp,
+        it returns the second of that timestamp.
+        If `expr` is a TIME type (since 4.1.0), it returns the second of the time-of-day.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT second('2018-02-14 12:58:59');
+           59
+          > SELECT second(TIME'13:25:59.999999');
+           59
+      
+  since: "1.5.0"
+
+- name: "session_window"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SessionWindow"
+  signatures:
+    - "session_window(time_column, gap_duration)"
+  usage: |2-
+    
+        session_window(time_column, gap_duration) - Generates session window given a timestamp specifying column and gap duration.
+          See 'Types of time windows' in Structured Streaming guide doc for detailed explanation and examples.
+      
+  arguments: |2-
+    
+        Arguments:
+          * time_column - The column or the expression to use as the timestamp for windowing by time. The time column must be of TimestampType.
+          * gap_duration - A string specifying the timeout of the session represented as "interval value"
+            (See Interval Literal for more details.) for the fixed gap duration, or
+            an expression which is applied for each input and evaluated to the "interval value" for the dynamic gap duration.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, session_window.start, session_window.end, count(*) as cnt FROM VALUES ('A1', '2021-01-01 00:00:00'), ('A1', '2021-01-01 00:04:30'), ('A1', '2021-01-01 00:10:00'), ('A2', '2021-01-01 00:01:00') AS tab(a, b) GROUP by a, session_window(b, '5 minutes') ORDER BY a, start;
+            A1	2021-01-01 00:00:00	2021-01-01 00:09:30	2
+            A1	2021-01-01 00:10:00	2021-01-01 00:15:00	1
+            A2	2021-01-01 00:01:00	2021-01-01 00:06:00	1
+          > SELECT a, session_window.start, session_window.end, count(*) as cnt FROM VALUES ('A1', '2021-01-01 00:00:00'), ('A1', '2021-01-01 00:04:30'), ('A1', '2021-01-01 00:10:00'), ('A2', '2021-01-01 00:01:00'), ('A2', '2021-01-01 00:04:30') AS tab(a, b) GROUP by a, session_window(b, CASE WHEN a = 'A1' THEN '5 minutes' WHEN a = 'A2' THEN '1 minute' ELSE '10 minutes' END) ORDER BY a, start;
+            A1	2021-01-01 00:00:00	2021-01-01 00:09:30	2
+            A1	2021-01-01 00:10:00	2021-01-01 00:15:00	1
+            A2	2021-01-01 00:01:00	2021-01-01 00:02:00	1
+            A2	2021-01-01 00:04:30	2021-01-01 00:05:30	1
+      
+  since: "3.2.0"
+
+- name: "time_bucket"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeBucketExpressionBuilder"
+  signatures:
+    - "time_bucket(bucketSize, ts[, origin])"
+  usage: |2-
+    
+        time_bucket(bucketSize, ts[, origin]) - Returns the start of the bucket that `ts` falls into,
+          where buckets are defined by the given `bucketSize` interval aligned to optional `origin`.
+          For `TIMESTAMP_NTZ`, bucketing is performed in UTC. For `TIMESTAMP`, year-month
+          interval buckets and calendar-day components of day-time interval buckets align
+          to the session time zone.
+      
+  arguments: |2-
+    
+        Arguments:
+          * bucketSize - A day-time or year-month interval defining the bucket size. Must be positive and foldable.
+          * ts - A TIMESTAMP or TIMESTAMP_NTZ value to bucket.
+          * origin - Optional TIMESTAMP or TIMESTAMP_NTZ alignment anchor. Defaults to 1970-01-01 00:00:00. Must be the same type as ts and must be foldable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_bucket(INTERVAL '15' MINUTE, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00');
+           2024-01-01 11:15:00
+          > SELECT time_bucket(INTERVAL '1' HOUR, TIMESTAMP '2024-01-01 11:27:00');
+           2024-01-01 11:00:00
+          > SELECT time_bucket(INTERVAL '1' MONTH, TIMESTAMP '2024-07-20 14:30:00', TIMESTAMP '2024-06-15 09:00:00');
+           2024-07-15 09:00:00
+      
+  since: "4.2.0"
+
+- name: "time_diff"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeDiff"
+  signatures:
+    - "time_diff(unit, start, end)"
+  usage: |2-
+    
+        time_diff(unit, start, end) - Gets the difference between the times in the specified units.
+      
+  arguments: |2-
+    
+        Arguments:
+          * unit - the unit of the difference between the given times
+              - "HOUR"
+              - "MINUTE"
+              - "SECOND"
+              - "MILLISECOND"
+              - "MICROSECOND"
+          * start - a starting TIME expression
+          * end - an ending TIME expression
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_diff('HOUR', TIME'20:30:29', TIME'21:30:28');
+           0
+          > SELECT time_diff('HOUR', TIME'20:30:29', TIME'21:30:29');
+           1
+          > SELECT time_diff('HOUR', TIME'20:30:29', TIME'12:00:00');
+           -8
+      
+  since: "4.1.0"
+
+- name: "time_from_micros"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeFromMicros"
+  signatures:
+    - "time_from_micros(micros)"
+  usage: "time_from_micros(micros) - Creates a TIME value from microseconds since midnight."
+  arguments: |2-
+    
+        Arguments:
+          * micros - microseconds since midnight (0 to 86399999999)
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_from_micros(0);
+           00:00:00
+          > SELECT time_from_micros(52200000000);
+           14:30:00
+          > SELECT time_from_micros(52200500000);
+           14:30:00.5
+          > SELECT time_from_micros(86399999999);
+           23:59:59.999999
+      
+  since: "4.2.0"
+
+- name: "time_from_millis"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeFromMillis"
+  signatures:
+    - "time_from_millis(millis)"
+  usage: "time_from_millis(millis) - Creates a TIME value from milliseconds since midnight."
+  arguments: |2-
+    
+        Arguments:
+          * millis - milliseconds since midnight (0 to 86399999)
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_from_millis(0);
+           00:00:00
+          > SELECT time_from_millis(52200000);
+           14:30:00
+          > SELECT time_from_millis(52200500);
+           14:30:00.5
+          > SELECT time_from_millis(86399999);
+           23:59:59.999
+      
+  since: "4.2.0"
+
+- name: "time_from_seconds"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeFromSeconds"
+  signatures:
+    - "time_from_seconds(seconds)"
+  usage: "time_from_seconds(seconds) - Creates a TIME value from seconds since midnight."
+  arguments: |2-
+    
+        Arguments:
+          * seconds - seconds since midnight (0 to 86399.999999).
+                      Supports decimals for fractional seconds.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_from_seconds(0);
+           00:00:00
+          > SELECT time_from_seconds(52200);
+           14:30:00
+          > SELECT time_from_seconds(52200.5);
+           14:30:00.5
+          > SELECT time_from_seconds(86399.999999);
+           23:59:59.999999
+      
+  since: "4.2.0"
+
+- name: "time_to_micros"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeToMicros"
+  signatures:
+    - "time_to_micros(time)"
+  usage: "time_to_micros(time) - Returns the number of microseconds since midnight for the given TIME value."
+  arguments: |2-
+    
+        Arguments:
+          * time - TIME value to convert
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_to_micros(TIME'00:00:00');
+           0
+          > SELECT time_to_micros(TIME'14:30:00');
+           52200000000
+          > SELECT time_to_micros(TIME'14:30:00.5');
+           52200500000
+          > SELECT time_to_micros(TIME'23:59:59.999999');
+           86399999999
+      
+  since: "4.2.0"
+
+- name: "time_to_millis"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeToMillis"
+  signatures:
+    - "time_to_millis(time)"
+  usage: "time_to_millis(time) - Returns the number of milliseconds since midnight for the given TIME value."
+  arguments: |2-
+    
+        Arguments:
+          * time - TIME value to convert
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_to_millis(TIME'00:00:00');
+           0
+          > SELECT time_to_millis(TIME'14:30:00');
+           52200000
+          > SELECT time_to_millis(TIME'14:30:00.5');
+           52200500
+          > SELECT time_to_millis(TIME'23:59:59.999');
+           86399999
+      
+  since: "4.2.0"
+
+- name: "time_to_seconds"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeToSeconds"
+  signatures:
+    - "time_to_seconds(time)"
+  usage: "time_to_seconds(time) - Returns the number of seconds since midnight for the given TIME value."
+  arguments: |2-
+    
+        Arguments:
+          * time - TIME value to convert
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_to_seconds(TIME'00:00:00');
+           0.000000
+          > SELECT time_to_seconds(TIME'14:30:00');
+           52200.000000
+          > SELECT time_to_seconds(TIME'14:30:00.5');
+           52200.500000
+          > SELECT time_to_seconds(TIME'23:59:59.999999');
+           86399.999999
+      
+  since: "4.2.0"
+
+- name: "time_trunc"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeTrunc"
+  signatures:
+    - "time_trunc(unit, time)"
+  usage: |2-
+    
+        time_trunc(unit, time) - Returns `time` truncated to the `unit`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * unit - the unit to truncate to
+              - "HOUR" - zero out the minutes and seconds with fraction part
+              - "MINUTE" - zero out the seconds with fraction part
+              - "SECOND" - zero out the fraction part of seconds
+              - "MILLISECOND" - zero out the microseconds
+              - "MICROSECOND" - zero out the nanoseconds
+          * time - a TIME expression
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT time_trunc('HOUR', TIME'09:32:05.359');
+           09:00:00
+          > SELECT time_trunc('MILLISECOND', TIME'09:32:05.123456');
+           09:32:05.123
+      
+  since: "4.1.0"
+
+- name: "timestamp_add"
+  scope: "pyspark_only"
+  signatures:
+    - "timestamp_add(unit, quantity, timestamp)"
+  usage: "timestamp_add(unit, quantity, timestamp) - Adds quantity units to timestamp."
+
+- name: "timestamp_diff"
+  scope: "pyspark_only"
+  signatures:
+    - "timestamp_diff(unit, start_timestamp, end_timestamp)"
+  usage: "timestamp_diff(unit, start_timestamp, end_timestamp) - Returns the difference between start_timestamp and end_timestamp in unit."
+
+- name: "timestamp_micros"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MicrosToTimestamp"
+  signatures:
+    - "timestamp_micros(microseconds)"
+  usage: "timestamp_micros(microseconds) - Creates timestamp from the number of microseconds since UTC epoch."
+  examples: |2-
+    
+        Examples:
+          > SELECT timestamp_micros(1230219000123123);
+           2008-12-25 07:30:00.123123
+      
+  since: "3.1.0"
+
+- name: "timestamp_millis"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MillisToTimestamp"
+  signatures:
+    - "timestamp_millis(milliseconds)"
+  usage: "timestamp_millis(milliseconds) - Creates timestamp from the number of milliseconds since UTC epoch."
+  examples: |2-
+    
+        Examples:
+          > SELECT timestamp_millis(1230219000123);
+           2008-12-25 07:30:00.123
+      
+  since: "3.1.0"
+
+- name: "timestamp_seconds"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SecondsToTimestamp"
+  signatures:
+    - "timestamp_seconds(seconds)"
+  usage: "timestamp_seconds(seconds) - Creates timestamp from the number of seconds (can be fractional) since UTC epoch."
+  examples: |2-
+    
+        Examples:
+          > SELECT timestamp_seconds(1230219000);
+           2008-12-25 07:30:00
+          > SELECT timestamp_seconds(1230219000.123);
+           2008-12-25 07:30:00.123
+      
+  since: "3.1.0"
+
+- name: "timestampadd"
+  scope: "spark_internal"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimestampAdd"
+  signatures:
+    - "timestampadd(unit, quantity, timestamp)"
+  usage: "timestampadd(unit, quantity, timestamp) - Adds quantity units to timestamp."
+
+- name: "timestampdiff"
+  scope: "spark_internal"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimestampDiff"
+  signatures:
+    - "timestampdiff(unit, start_timestamp, end_timestamp)"
+  usage: "timestampdiff(unit, start_timestamp, end_timestamp) - Returns the difference between start_timestamp and end_timestamp in unit."
+
+- name: "to_date"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ParseToDate"
+  signatures:
+    - "to_date(date_str[, fmt])"
+  usage: |2-
+    
+        to_date(date_str[, fmt]) - Parses the `date_str` expression with the `fmt` expression to
+          a date. Returns null with invalid input. By default, it follows casting rules to a date if
+          the `fmt` is omitted.
+      
+  arguments: |2-
+    
+        Arguments:
+          * date_str - A string to be parsed to date.
+          * fmt - Date format pattern to follow. See Datetime Patterns for valid
+                  date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_date('2009-07-30 04:17:52');
+           2009-07-30
+          > SELECT to_date('2016-12-31', 'yyyy-MM-dd');
+           2016-12-31
+      
+  since: "1.5.0"
+
+- name: "to_time"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToTime"
+  signatures:
+    - "to_time(str[, format])"
+  usage: |2-
+    
+        to_time(str[, format]) - Parses the `str` expression with the `format` expression to a time.
+        If `format` is malformed or its application does not result in a well formed time, the function
+        raises an error. By default, it follows casting rules to a time if the `format` is omitted.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - A string to be parsed to time.
+          * format - Time format pattern to follow. See Datetime Patterns for valid
+                     time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_time('00:12:00');
+           00:12:00
+          > SELECT to_time('12.10.05', 'HH.mm.ss');
+           12:10:05
+      
+  since: "4.1.0"
+
+- name: "to_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ParseToTimestamp"
+  signatures:
+    - "to_timestamp(timestamp_str[, fmt])"
+  usage: |2-
+    
+        to_timestamp(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
+          to a timestamp. Returns null with invalid input. By default, it follows casting rules to
+          a timestamp if the `fmt` is omitted. The result data type is consistent with the value of
+          configuration `spark.sql.timestampType`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * timestamp_str - A string to be parsed to timestamp.
+          * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid
+                  date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_timestamp('2016-12-31 00:12:00');
+           2016-12-31 00:12:00
+          > SELECT to_timestamp('2016-12-31', 'yyyy-MM-dd');
+           2016-12-31 00:00:00
+      
+  since: "2.2.0"
+
+- name: "to_timestamp_ltz"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ParseToTimestampLTZExpressionBuilder"
+  signatures:
+    - "to_timestamp_ltz(timestamp_str[, fmt])"
+  usage: |2-
+    
+        to_timestamp_ltz(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
+          to a timestamp with local time zone. Returns null with invalid input. By default, it follows casting rules to
+          a timestamp if the `fmt` is omitted.
+      
+  arguments: |2-
+    
+        Arguments:
+          * timestamp_str - A string to be parsed to timestamp with local time zone.
+          * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid
+                  date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_timestamp_ltz('2016-12-31 00:12:00');
+           2016-12-31 00:12:00
+          > SELECT to_timestamp_ltz('2016-12-31', 'yyyy-MM-dd');
+           2016-12-31 00:00:00
+      
+  since: "3.4.0"
+
+- name: "to_timestamp_ntz"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ParseToTimestampNTZExpressionBuilder"
+  signatures:
+    - "to_timestamp_ntz(timestamp_str[, fmt])"
+  usage: |2-
+    
+        to_timestamp_ntz(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
+          to a timestamp without time zone. Returns null with invalid input. By default, it follows casting rules to
+          a timestamp if the `fmt` is omitted.
+      
+  arguments: |2-
+    
+        Arguments:
+          * timestamp_str - A string to be parsed to timestamp without time zone.
+          * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid
+                  date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_timestamp_ntz('2016-12-31 00:12:00');
+           2016-12-31 00:12:00
+          > SELECT to_timestamp_ntz('2016-12-31', 'yyyy-MM-dd');
+           2016-12-31 00:00:00
+      
+  since: "3.4.0"
+
+- name: "to_unix_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToUnixTimestamp"
+  signatures:
+    - "to_unix_timestamp(timeExp[, fmt])"
+  usage: "to_unix_timestamp(timeExp[, fmt]) - Returns the UNIX timestamp of the given time."
+  arguments: |2-
+    
+        Arguments:
+          * timeExp - A date/timestamp or string which is returned as a UNIX timestamp.
+          * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
+                  Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns
+                  for valid date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd');
+           1460098800
+      
+  since: "1.6.0"
+
+- name: "to_utc_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp"
+  signatures:
+    - "to_utc_timestamp(timestamp, timezone)"
+  usage: "to_utc_timestamp(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'."
+  examples: |2-
+    
+        Examples:
+          > SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul');
+           2016-08-30 15:00:00
+      
+  since: "1.5.0"
+
+- name: "trunc"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TruncDate"
+  signatures:
+    - "trunc(date, fmt)"
+  usage: |2-
+    
+        trunc(date, fmt) - Returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * date - date value or valid date string
+          * fmt - the format representing the unit to be truncated to
+              - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in
+              - "QUARTER" - truncate to the first date of the quarter that the `date` falls in
+              - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in
+              - "WEEK" - truncate to the Monday of the week that the `date` falls in
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT trunc('2019-08-04', 'week');
+           2019-07-29
+          > SELECT trunc('2019-08-04', 'quarter');
+           2019-07-01
+          > SELECT trunc('2009-02-12', 'MM');
+           2009-02-01
+          > SELECT trunc('2015-10-27', 'YEAR');
+           2015-01-01
+      
+  since: "1.5.0"
+
+- name: "try_make_interval"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryMakeInterval"
+  signatures:
+    - "try_make_interval([years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]])"
+  usage: "try_make_interval([years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]]) - This is a special version of `make_interval` that performs the same operation, but returns NULL when an overflow occurs."
+  arguments: |2-
+    
+        Arguments:
+          * years - the number of years, positive or negative
+          * months - the number of months, positive or negative
+          * weeks - the number of weeks, positive or negative
+          * days - the number of days, positive or negative
+          * hours - the number of hours, positive or negative
+          * mins - the number of minutes, positive or negative
+          * secs - the number of seconds with the fractional part in microsecond precision.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_make_interval(100, 11, 1, 1, 12, 30, 01.001001);
+           100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds
+          > SELECT try_make_interval(100, null, 3);
+           NULL
+          > SELECT try_make_interval(0, 1, 0, 1, 0, 0, 100.000001);
+           1 months 1 days 1 minutes 40.000001 seconds
+          > SELECT try_make_interval(2147483647);
+           NULL
+      
+  since: "4.0.0"
+
+- name: "try_make_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryMakeTimestampExpressionBuilder"
+  signatures:
+    - "try_make_timestamp(year, month, day, hour, min, sec[, timezone])"
+    - "try_make_timestamp(date[, time[, timezone]])"
+  usage: |2-
+    
+        try_make_timestamp(year, month, day, hour, min, sec[, timezone]) - Try to create a timestamp from year, month, day, hour, min, sec and timezone fields. The result data type is consistent with the value of configuration `spark.sql.timestampType`. The function returns NULL on invalid inputs.
+    
+        try_make_timestamp(date[, time[, timezone]]) - Try to create a timestamp from date, time, and timezone fields.
+        
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+          * hour - the hour-of-day to represent, from 0 to 23
+          * min - the minute-of-hour to represent, from 0 to 59
+          * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60.
+                  The value can be either an integer like 13 , or a fraction like 13.123.
+                  If the sec argument equals to 60, the seconds field is set
+                  to 0 and 1 minute is added to the final timestamp.
+          * date - a date expression
+          * time - a time expression (optional). Default is 00:00:00.
+          * timezone - the time zone identifier (optional). For example, CET, UTC and etc.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_make_timestamp(2014, 12, 28, 6, 30, 45.887);
+           2014-12-28 06:30:45.887
+          > SELECT try_make_timestamp(2014, 12, 28, 6, 30, 45.887, 'CET');
+           2014-12-27 21:30:45.887
+          > SELECT try_make_timestamp(DATE'2014-12-28');
+           2014-12-28 00:00:00
+          > SELECT try_make_timestamp(DATE'2014-12-28', TIME'6:30:45.887');
+           2014-12-28 06:30:45.887
+          > SELECT try_make_timestamp(DATE'2014-12-28', TIME'6:30:45.887', 'CET');
+           2014-12-27 21:30:45.887
+          > SELECT try_make_timestamp(2019, 6, 30, 23, 59, 60);
+           2019-07-01 00:00:00
+          > SELECT try_make_timestamp(2019, 6, 30, 23, 59, 1);
+           2019-06-30 23:59:01
+          > SELECT try_make_timestamp(null, 7, 22, 15, 30, 0);
+           NULL
+          > SELECT try_make_timestamp(2024, 13, 22, 15, 30, 0);
+           NULL
+      
+  since: "4.0.0"
+
+- name: "try_make_timestamp_ltz"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryMakeTimestampLTZExpressionBuilder"
+  signatures:
+    - "try_make_timestamp_ltz(year, month, day, hour, min, sec[, timezone])"
+    - "try_make_timestamp_ltz(date, time[, timezone])"
+  usage: |2-
+    
+        try_make_timestamp_ltz(year, month, day, hour, min, sec[, timezone]) - Try to create the current timestamp with local time zone from year, month, day, hour, min, sec and (optional) timezone fields. The function returns NULL on invalid inputs.
+    
+        try_make_timestamp_ltz(date, time[, timezone]) - Try to create the current timestamp with local time zone from date, time and (optional) timezone fields.
+        
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+          * hour - the hour-of-day to represent, from 0 to 23
+          * min - the minute-of-hour to represent, from 0 to 59
+          * sec - the second-of-minute and its micro-fraction to represent, from
+                  0 to 60. If the sec argument equals to 60, the seconds field is set
+                  to 0 and 1 minute is added to the final timestamp.
+          * timezone - the time zone identifier. For example, CET, UTC and etc.
+          * date - a date to represent, from 0001-01-01 to 9999-12-31
+          * time - a local time to represent, from 00:00:00 to 23:59:59.999999
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887);
+           2014-12-28 06:30:45.887
+          > SELECT try_make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887, 'CET');
+           2014-12-27 21:30:45.887
+          > SELECT try_make_timestamp_ltz(2019, 6, 30, 23, 59, 60);
+           2019-07-01 00:00:00
+          > SELECT try_make_timestamp_ltz(null, 7, 22, 15, 30, 0);
+           NULL
+          > SELECT try_make_timestamp_ltz(2024, 13, 22, 15, 30, 0);
+           NULL
+          > SELECT try_make_timestamp_ltz(DATE'2014-12-28', TIME'6:30:45.887');
+           2014-12-28 06:30:45.887
+          > SELECT try_make_timestamp_ltz(DATE'2014-12-28', TIME'6:30:45.887', 'CET');
+           2014-12-27 21:30:45.887
+      
+  since: "4.0.0"
+
+- name: "try_make_timestamp_ntz"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryMakeTimestampNTZExpressionBuilder"
+  signatures:
+    - "try_make_timestamp_ntz(year, month, day, hour, min, sec)"
+    - "try_make_timestamp_ntz(date, time)"
+  usage: |2-
+    
+        try_make_timestamp_ntz(year, month, day, hour, min, sec) - Try to create local date-time from year, month, day, hour, min, sec fields. The function returns NULL on invalid inputs.
+    
+        try_make_timestamp_ntz(date, time) - Create a local date-time from date and time fields.
+        
+  arguments: |2-
+    
+        Arguments:
+          * year - the year to represent, from 1 to 9999
+          * month - the month-of-year to represent, from 1 (January) to 12 (December)
+          * day - the day-of-month to represent, from 1 to 31
+          * hour - the hour-of-day to represent, from 0 to 23
+          * min - the minute-of-hour to represent, from 0 to 59
+          * sec - the second-of-minute and its micro-fraction to represent, from
+                  0 to 60. If the sec argument equals to 60, the seconds field is set
+                  to 0 and 1 minute is added to the final timestamp.
+          * date - a date to represent, from 0001-01-01 to 9999-12-31
+          * time - a local time to represent, from 00:00:00 to 23:59:59.999999
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887);
+           2014-12-28 06:30:45.887
+          > SELECT try_make_timestamp_ntz(2019, 6, 30, 23, 59, 60);
+           2019-07-01 00:00:00
+          > SELECT try_make_timestamp_ntz(null, 7, 22, 15, 30, 0);
+           NULL
+          > SELECT try_make_timestamp_ntz(2024, 13, 22, 15, 30, 0);
+           NULL
+          > SELECT try_make_timestamp_ntz(DATE'2014-12-28', TIME'6:30:45.887');
+           2014-12-28 06:30:45.887
+      
+  since: "4.0.0"
+
+- name: "try_to_time"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryToTimeExpressionBuilder"
+  signatures:
+    - "try_to_time(str[, format])"
+  usage: |2-
+    
+        try_to_time(str[, format]) - Parses the `str` expression with the `format` expression to a time.
+        If `format` is malformed or its application does not result in a well formed time, the function
+        returns NULL. By default, it follows casting rules to a time if the `format` is omitted.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - A string to be parsed to time.
+          * format - Time format pattern to follow. See Datetime Patterns for valid
+                     time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_to_time('00:12:00.001');
+           00:12:00.001
+          > SELECT try_to_time('12.10.05.999999', 'HH.mm.ss.SSSSSS');
+           12:10:05.999999
+          > SELECT try_to_time('foo', 'HH:mm:ss');
+           NULL
+      
+  since: "4.1.0"
+
+- name: "try_to_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryToTimestampExpressionBuilder"
+  signatures:
+    - "try_to_timestamp(timestamp_str[, fmt])"
+  usage: |2-
+    
+        try_to_timestamp(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
+          to a timestamp. The function always returns null on an invalid input with/without ANSI SQL
+          mode enabled. By default, it follows casting rules to a timestamp if the `fmt` is omitted.
+          The result data type is consistent with the value of configuration `spark.sql.timestampType`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * timestamp_str - A string to be parsed to timestamp.
+          * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid
+                  date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_to_timestamp('2016-12-31 00:12:00');
+           2016-12-31 00:12:00
+          > SELECT try_to_timestamp('2016-12-31', 'yyyy-MM-dd');
+           2016-12-31 00:00:00
+          > SELECT try_to_timestamp('foo', 'yyyy-MM-dd');
+           NULL
+      
+  since: "3.4.0"
+
+- name: "unix_date"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnixDate"
+  signatures:
+    - "unix_date(date)"
+  usage: "unix_date(date) - Returns the number of days since 1970-01-01."
+  examples: |2-
+    
+        Examples:
+          > SELECT unix_date(DATE("1970-01-02"));
+           1
+      
+  since: "3.1.0"
+
+- name: "unix_micros"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnixMicros"
+  signatures:
+    - "unix_micros(timestamp)"
+  usage: "unix_micros(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC."
+  examples: |2-
+    
+        Examples:
+          > SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z'));
+           1000000
+      
+  since: "3.1.0"
+
+- name: "unix_millis"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnixMillis"
+  signatures:
+    - "unix_millis(timestamp)"
+  usage: "unix_millis(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision."
+  examples: |2-
+    
+        Examples:
+          > SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z'));
+           1000
+      
+  since: "3.1.0"
+
+- name: "unix_seconds"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnixSeconds"
+  signatures:
+    - "unix_seconds(timestamp)"
+  usage: "unix_seconds(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision."
+  examples: |2-
+    
+        Examples:
+          > SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'));
+           1
+      
+  since: "3.1.0"
+
+- name: "unix_timestamp"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnixTimestamp"
+  signatures:
+    - "unix_timestamp([timeExp[, fmt]])"
+  usage: "unix_timestamp([timeExp[, fmt]]) - Returns the UNIX timestamp of current or specified time."
+  arguments: |2-
+    
+        Arguments:
+          * timeExp - A date/timestamp or string. If not provided, this defaults to current time.
+          * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
+                  Default value is "yyyy-MM-dd HH:mm:ss". See  Datetime Patterns
+                  for valid date and time format patterns.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT unix_timestamp();
+           1476884637
+          > SELECT unix_timestamp('2016-04-08', 'yyyy-MM-dd');
+           1460041200
+      
+  since: "1.5.0"
+
+- name: "weekday"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.WeekDay"
+  signatures:
+    - "weekday(date)"
+  usage: "weekday(date) - Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday)."
+  examples: |2-
+    
+        Examples:
+          > SELECT weekday('2009-07-30');
+           3
+      
+  since: "2.4.0"
+
+- name: "weekofyear"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.WeekOfYear"
+  signatures:
+    - "weekofyear(date)"
+  usage: "weekofyear(date) - Returns the week of the year of the given date. A week is considered to start on a Monday and week 1 is the first week with >3 days."
+  examples: |2-
+    
+        Examples:
+          > SELECT weekofyear('2008-02-20');
+           8
+      
+  since: "1.5.0"
+
+- name: "window"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TimeWindow"
+  signatures:
+    - "window(time_column, window_duration[, slide_duration[, start_time]])"
+  usage: |2-
+    
+        window(time_column, window_duration[, slide_duration[, start_time]]) - Bucketize rows into one or more time windows given a timestamp specifying column.
+          Window starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) but not in [12:00,12:05).
+          Windows can support microsecond precision. Windows in the order of months are not supported.
+          See 'Window Operations on Event Time' in Structured Streaming guide doc for detailed explanation and examples.
+      
+  arguments: |2-
+    
+        Arguments:
+          * time_column - The column or the expression to use as the timestamp for windowing by time. The time column must be of TimestampType.
+          * window_duration - A string specifying the width of the window represented as "interval value".
+            (See Interval Literal for more details.)
+            Note that the duration is a fixed length of time, and does not vary over time according to a calendar.
+          * slide_duration - A string specifying the sliding interval of the window represented as "interval value".
+            A new window will be generated every `slide_duration`. Must be less than or equal to the `window_duration`.
+            This duration is likewise absolute, and does not vary according to a calendar.
+          * start_time - The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals.
+            For example, in order to have hourly tumbling windows that start 15 minutes past the hour,
+            e.g. 12:15-13:15, 13:15-14:15... provide `start_time` as `15 minutes`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, window.start, window.end, count(*) as cnt FROM VALUES ('A1', '2021-01-01 00:00:00'), ('A1', '2021-01-01 00:04:30'), ('A1', '2021-01-01 00:06:00'), ('A2', '2021-01-01 00:01:00') AS tab(a, b) GROUP by a, window(b, '5 minutes') ORDER BY a, start;
+            A1	2021-01-01 00:00:00	2021-01-01 00:05:00	2
+            A1	2021-01-01 00:05:00	2021-01-01 00:10:00	1
+            A2	2021-01-01 00:00:00	2021-01-01 00:05:00	1
+          > SELECT a, window.start, window.end, count(*) as cnt FROM VALUES ('A1', '2021-01-01 00:00:00'), ('A1', '2021-01-01 00:04:30'), ('A1', '2021-01-01 00:06:00'), ('A2', '2021-01-01 00:01:00') AS tab(a, b) GROUP by a, window(b, '10 minutes', '5 minutes') ORDER BY a, start;
+            A1	2020-12-31 23:55:00	2021-01-01 00:05:00	2
+            A1	2021-01-01 00:00:00	2021-01-01 00:10:00	3
+            A1	2021-01-01 00:05:00	2021-01-01 00:15:00	1
+            A2	2020-12-31 23:55:00	2021-01-01 00:05:00	1
+            A2	2021-01-01 00:00:00	2021-01-01 00:10:00	1
+      
+  since: "2.0.0"
+
+- name: "window_time"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.WindowTime"
+  signatures:
+    - "window_time(window_column)"
+  usage: |2-
+    
+        window_time(window_column) - Extract the time value from time/session window column which can be used for event time value of window.
+          The extracted time is (window.end - 1) which reflects the fact that the aggregating
+          windows have exclusive upper bound - [start, end)
+          See 'Window Operations on Event Time' in Structured Streaming guide doc for detailed explanation and examples.
+      
+  arguments: |2-
+    
+        Arguments:
+          * window_column - The column representing time/session window.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, window.start as start, window.end as end, window_time(window), cnt FROM (SELECT a, window, count(*) as cnt FROM VALUES ('A1', '2021-01-01 00:00:00'), ('A1', '2021-01-01 00:04:30'), ('A1', '2021-01-01 00:06:00'), ('A2', '2021-01-01 00:01:00') AS tab(a, b) GROUP by a, window(b, '5 minutes') ORDER BY a, window.start);
+            A1	2021-01-01 00:00:00	2021-01-01 00:05:00	2021-01-01 00:04:59.999999	2
+            A1	2021-01-01 00:05:00	2021-01-01 00:10:00	2021-01-01 00:09:59.999999	1
+            A2	2021-01-01 00:00:00	2021-01-01 00:05:00	2021-01-01 00:04:59.999999	1
+      
+  since: "3.4.0"
+
+- name: "year"
+  scope: "sql_builtin"
+  group: "datetime_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Year"
+  signatures:
+    - "year(date)"
+  usage: "year(date) - Returns the year component of the date/timestamp."
+  examples: |2-
+    
+        Examples:
+          > SELECT year('2016-07-30');
+           2016
+      
+  since: "1.5.0"
+
+- name: "years"
+  scope: "spark_internal"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Years"
+  signatures:
+    - "years(date)"
+  usage: "years(date) - Returns the year component of the date/timestamp."
diff --git a/crates/sail-plan/data/functions/scalar/geo.yaml b/crates/sail-plan/data/functions/scalar/geo.yaml
new file mode 100644
index 000000000..745d49668
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/geo.yaml
@@ -0,0 +1,123 @@
+- name: "st_asbinary"
+  scope: "sql_builtin"
+  group: "st_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.st.ST_AsBinary"
+  signatures:
+    - "st_asbinary(geo[, endianness])"
+  usage: "st_asbinary(geo[, endianness]) - Returns the geospatial value (value of type GEOGRAPHY or GEOMETRY) in WKB format using the specified endianness ('NDR' for little-endian, 'XDR' for big-endian), if provided. Defaults to little-endian encoding."
+  arguments: |2-
+    
+        Arguments:
+          * geo - A geospatial value, either a GEOGRAPHY or a GEOMETRY.
+          * endianness - The optional endianness of the output WKB, 'NDR' for little-endian (default)
+                         or 'XDR' for big-endian.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT hex(st_asbinary(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040')));
+           0101000000000000000000F03F0000000000000040
+          > SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'), 'XDR'));
+           00000000013FF00000000000004000000000000000
+      
+  since: "4.1.0"
+
+- name: "st_geogfromwkb"
+  scope: "sql_builtin"
+  group: "st_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.st.ST_GeogFromWKB"
+  signatures:
+    - "st_geogfromwkb(wkb)"
+  usage: "st_geogfromwkb(wkb) - Parses the WKB description of a geography and returns the corresponding GEOGRAPHY value."
+  arguments: |2-
+    
+        Arguments:
+          * wkb - A BINARY value in WKB format, representing a GEOGRAPHY value.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT hex(st_asbinary(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040')));
+           0101000000000000000000F03F0000000000000040
+      
+  since: "4.1.0"
+
+- name: "st_geomfromwkb"
+  scope: "sql_builtin"
+  group: "st_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.st.ST_GeomFromWKB"
+  signatures:
+    - "st_geomfromwkb(wkb[, srid])"
+  usage: "st_geomfromwkb(wkb[, srid]) - Parses the WKB description of a geometry and returns the corresponding GEOMETRY value."
+  arguments: |2-
+    
+        Arguments:
+          * wkb - A BINARY value in WKB format, representing a GEOMETRY value.
+          * srid - The optional SRID value of the geometry. Default is 0.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040')));
+           0101000000000000000000F03F0000000000000040
+          > SELECT st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'));
+           0
+          > SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326)));
+           0101000000000000000000F03F0000000000000040
+          > SELECT st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326));
+           4326
+      
+  since: "4.1.0"
+
+- name: "st_setsrid"
+  scope: "sql_builtin"
+  group: "st_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.st.ST_SetSrid"
+  signatures:
+    - "st_setsrid(geo, srid)"
+  usage: "st_setsrid(geo, srid) - Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value."
+  arguments: |2-
+    
+        Arguments:
+          * geo - A GEOGRAPHY or GEOMETRY value.
+          * srid - The new SRID value of the geography or geometry.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT st_srid(st_setsrid(ST_GeogFromWKB(X'0101000000000000000000F03F0000000000000040'), 4326));
+           4326
+          > SELECT st_srid(st_setsrid(ST_GeomFromWKB(X'0101000000000000000000F03F0000000000000040'), 3857));
+           3857
+      
+  since: "4.1.0"
+
+- name: "st_srid"
+  scope: "sql_builtin"
+  group: "st_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.st.ST_Srid"
+  signatures:
+    - "st_srid(geo)"
+  usage: "st_srid(geo) - Returns the SRID of the input GEOGRAPHY or GEOMETRY value."
+  arguments: |2-
+    
+        Arguments:
+          * geo - A GEOGRAPHY or GEOMETRY value.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT st_srid(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040'));
+           4326
+          > SELECT st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'));
+           0
+          > SELECT st_srid(NULL);
+           NULL
+      
+  since: "4.1.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/hash.yaml b/crates/sail-plan/data/functions/scalar/hash.yaml
new file mode 100644
index 000000000..f25070fc0
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/hash.yaml
@@ -0,0 +1,116 @@
+- name: "crc32"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Crc32"
+  signatures:
+    - "crc32(expr)"
+  usage: "crc32(expr) - Returns a cyclic redundancy check value of the `expr` as a bigint."
+  examples: |2-
+    
+        Examples:
+          > SELECT crc32('Spark');
+           1557323817
+      
+  since: "1.5.0"
+
+- name: "hash"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Murmur3Hash"
+  signatures:
+    - "hash(expr1, expr2, ...)"
+  usage: "hash(expr1, expr2, ...) - Returns a hash value of the arguments."
+  examples: |2-
+    
+        Examples:
+          > SELECT hash('Spark', array(123), 2);
+           -1321691492
+      
+  since: "2.0.0"
+
+- name: "md5"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Md5"
+  signatures:
+    - "md5(expr)"
+  usage: "md5(expr) - Returns an MD5 128-bit checksum as a hex string of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT md5('Spark');
+           8cde774d6f7333752ed72cacddb05126
+      
+  since: "1.5.0"
+
+- name: "sha"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sha1"
+  signatures:
+    - "sha(expr)"
+  usage: "sha(expr) - Returns a sha1 hash value as a hex string of the `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT sha('Spark');
+           85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c
+      
+  since: "1.5.0"
+
+- name: "sha1"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sha1"
+  signatures:
+    - "sha1(expr)"
+  usage: "sha1(expr) - Returns a sha1 hash value as a hex string of the `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT sha1('Spark');
+           85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c
+      
+  since: "1.5.0"
+
+- name: "sha2"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sha2"
+  signatures:
+    - "sha2(expr, bitLength)"
+  usage: |2-
+    
+        sha2(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of `expr`.
+          SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sha2('Spark', 256);
+           529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b
+      
+  since: "1.5.0"
+
+- name: "xxhash64"
+  scope: "sql_builtin"
+  group: "hash_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.XxHash64"
+  signatures:
+    - "xxhash64(expr1, expr2, ...)"
+  usage: "xxhash64(expr1, expr2, ...) - Returns a 64-bit hash value of the arguments. Hash seed is 42."
+  examples: |2-
+    
+        Examples:
+          > SELECT xxhash64('Spark', array(123), 2);
+           5602566077635097486
+      
+  since: "3.0.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/json.yaml b/crates/sail-plan/data/functions/scalar/json.yaml
new file mode 100644
index 000000000..5326e72d7
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/json.yaml
@@ -0,0 +1,157 @@
+- name: "from_json"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.JsonToStructs"
+  signatures:
+    - "from_json(jsonStr, schema[, options])"
+  usage: "from_json(jsonStr, schema[, options]) - Returns a struct value with the given `jsonStr` and `schema`."
+  examples: |2-
+    
+        Examples:
+          > SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE');
+           {"a":1,"b":0.8}
+          > SELECT from_json('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
+           {"time":2015-08-26 00:00:00}
+          > SELECT from_json('{"teacher": "Alice", "student": [{"name": "Bob", "rank": 1}, {"name": "Charlie", "rank": 2}]}', 'STRUCT>>');
+           {"teacher":"Alice","student":[{"name":"Bob","rank":1},{"name":"Charlie","rank":2}]}
+      
+  since: "2.2.0"
+
+- name: "get_json_object"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.GetJsonObject"
+  signatures:
+    - "get_json_object(json_txt, path)"
+  usage: "get_json_object(json_txt, path) - Extracts a json object from `path`."
+  examples: |2-
+    
+        Examples:
+          > SELECT get_json_object('{"a":"b"}', '$.a');
+           b
+          > SELECT get_json_object('[{"a":"b"},{"a":"c"}]', '$[0].a');
+           b
+          > SELECT get_json_object('[{"a":"b"},{"a":"c"}]', '$[*].a');
+           ["b","c"]
+      
+  since: "1.5.0"
+
+- name: "json_array_length"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.LengthOfJsonArray"
+  signatures:
+    - "json_array_length(jsonArray)"
+  usage: "json_array_length(jsonArray) - Returns the number of elements in the outermost JSON array."
+  arguments: |2-
+    
+        Arguments:
+          * jsonArray - A JSON array. `NULL` is returned in case of any other valid JSON string,
+              `NULL` or an invalid JSON.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT json_array_length('[1,2,3,4]');
+            4
+          > SELECT json_array_length('[1,2,3,{"f1":1,"f2":[5,6]},4]');
+            5
+          > SELECT json_array_length('[1,2');
+            NULL
+      
+  since: "3.1.0"
+
+- name: "json_object_keys"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.JsonObjectKeys"
+  signatures:
+    - "json_object_keys(json_object)"
+  usage: "json_object_keys(json_object) - Returns all the keys of the outermost JSON object as an array."
+  arguments: |2-
+    
+        Arguments:
+          * json_object - A JSON object. If a valid JSON object is given, all the keys of the outermost
+              object will be returned as an array. If it is any other valid JSON string, an invalid JSON
+              string or an empty string, the function returns null.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT json_object_keys('{}');
+            []
+          > SELECT json_object_keys('{"key": "value"}');
+            ["key"]
+          > SELECT json_object_keys('{"f1":"abc","f2":{"f3":"a", "f4":"b"}}');
+            ["f1","f2"]
+      
+  since: "3.1.0"
+
+- name: "json_tuple"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.JsonTuple"
+  signatures:
+    - "json_tuple(jsonStr, p1, p2, ..., pn)"
+  usage: "json_tuple(jsonStr, p1, p2, ..., pn) - Returns a tuple like the function get_json_object, but it takes multiple names. All the input parameters and output column types are string."
+  examples: |2-
+    
+        Examples:
+          > SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');
+           1	2
+      
+  since: "1.6.0"
+
+- name: "schema_of_json"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SchemaOfJson"
+  signatures:
+    - "schema_of_json(json[, options])"
+  usage: "schema_of_json(json[, options]) - Returns schema in the DDL format of JSON string."
+  examples: |2-
+    
+        Examples:
+          > SELECT schema_of_json('[{"col":0}]');
+           ARRAY>
+          > SELECT schema_of_json('[{"col":01}]', map('allowNumericLeadingZeros', 'true'));
+           ARRAY>
+      
+  since: "2.4.0"
+
+- name: "to_json"
+  scope: "sql_builtin"
+  group: "json_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StructsToJson"
+  signatures:
+    - "to_json(expr[, options])"
+  usage: "to_json(expr[, options]) - Returns a JSON string with a given struct value"
+  examples: |2-
+    
+        Examples:
+          > SELECT to_json(named_struct('a', 1, 'b', 2));
+           {"a":1,"b":2}
+          > SELECT to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));
+           {"time":"26/08/2015"}
+          > SELECT to_json(array(named_struct('a', 1, 'b', 2)));
+           [{"a":1,"b":2}]
+          > SELECT to_json(map('a', named_struct('b', 1)));
+           {"a":{"b":1}}
+          > SELECT to_json(map(named_struct('a', 1),named_struct('b', 2)));
+           {"[1]":{"b":2}}
+          > SELECT to_json(map('a', 1));
+           {"a":1}
+          > SELECT to_json(array(map('a', 1)));
+           [{"a":1}]
+          > SELECT to_json(named_struct('b', 1, 'a', 2), map('sortKeys', 'true'));
+           {"a":2,"b":1}
+      
+  since: "2.2.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/map.yaml b/crates/sail-plan/data/functions/scalar/map.yaml
new file mode 100644
index 000000000..6135f26f9
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/map.yaml
@@ -0,0 +1,151 @@
+- name: "map"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CreateMap"
+  signatures:
+    - "map(key0, value0, key1, value1, ...)"
+  usage: "map(key0, value0, key1, value1, ...) - Creates a map with the given key/value pairs."
+  examples: |2-
+    
+        Examples:
+          > SELECT map(1.0, '2', 3.0, '4');
+           {1.0:"2",3.0:"4"}
+      
+  since: "2.0.0"
+
+- name: "map_concat"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapConcat"
+  signatures:
+    - "map_concat(map, ...)"
+  usage: "map_concat(map, ...) - Returns the union of all the given maps"
+  examples: |2-
+    
+        Examples:
+          > SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c'));
+           {1:"a",2:"b",3:"c"}
+      
+  since: "2.4.0"
+
+- name: "map_contains_key"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapContainsKey"
+  signatures:
+    - "map_contains_key(map, key)"
+  usage: "map_contains_key(map, key) - Returns true if the map contains the key."
+  examples: |2-
+    
+        Examples:
+          > SELECT map_contains_key(map(1, 'a', 2, 'b'), 1);
+           true
+          > SELECT map_contains_key(map(1, 'a', 2, 'b'), 3);
+           false
+      
+  since: "3.3.0"
+
+- name: "map_entries"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapEntries"
+  signatures:
+    - "map_entries(map)"
+  usage: "map_entries(map) - Returns an unordered array of all entries in the given map."
+  examples: |2-
+    
+        Examples:
+          > SELECT map_entries(map(1, 'a', 2, 'b'));
+           [{"key":1,"value":"a"},{"key":2,"value":"b"}]
+      
+  since: "3.0.0"
+
+- name: "map_from_arrays"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapFromArrays"
+  signatures:
+    - "map_from_arrays(keys, values)"
+  usage: |2-
+    
+        map_from_arrays(keys, values) - Creates a map with a pair of the given key/value arrays. All elements
+          in keys should not be null
+  examples: |2-
+    
+        Examples:
+          > SELECT map_from_arrays(array(1.0, 3.0), array('2', '4'));
+           {1.0:"2",3.0:"4"}
+      
+  since: "2.4.0"
+
+- name: "map_from_entries"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapFromEntries"
+  signatures:
+    - "map_from_entries(arrayOfEntries)"
+  usage: "map_from_entries(arrayOfEntries) - Returns a map created from the given array of entries."
+  examples: |2-
+    
+        Examples:
+          > SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b')));
+           {1:"a",2:"b"}
+      
+  since: "2.4.0"
+
+- name: "map_keys"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapKeys"
+  signatures:
+    - "map_keys(map)"
+  usage: "map_keys(map) - Returns an unordered array containing the keys of the map."
+  examples: |2-
+    
+        Examples:
+          > SELECT map_keys(map(1, 'a', 2, 'b'));
+           [1,2]
+      
+  since: "2.0.0"
+
+- name: "map_values"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MapValues"
+  signatures:
+    - "map_values(map)"
+  usage: "map_values(map) - Returns an unordered array containing the values of the map."
+  examples: |2-
+    
+        Examples:
+          > SELECT map_values(map(1, 'a', 2, 'b'));
+           ["a","b"]
+      
+  since: "2.0.0"
+
+- name: "str_to_map"
+  scope: "sql_builtin"
+  group: "map_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringToMap"
+  signatures:
+    - "str_to_map(text[, pairDelim[, keyValueDelim]])"
+  usage: "str_to_map(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for `pairDelim` and ':' for `keyValueDelim`. Both `pairDelim` and `keyValueDelim` are treated as regular expressions."
+  examples: |2-
+    
+        Examples:
+          > SELECT str_to_map('a:1,b:2,c:3', ',', ':');
+           {"a":"1","b":"2","c":"3"}
+          > SELECT str_to_map('a');
+           {"a":null}
+      
+  since: "2.0.1"
+
diff --git a/crates/sail-plan/data/functions/scalar/math.yaml b/crates/sail-plan/data/functions/scalar/math.yaml
new file mode 100644
index 000000000..85661c828
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/math.yaml
@@ -0,0 +1,1349 @@
+- name: "%"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Remainder"
+  signatures:
+    - "expr1 % expr2, or mod(expr1, expr2)"
+  usage: "expr1 % expr2, or mod(expr1, expr2) - Returns the remainder after `expr1`/`expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 % 1.8;
+           0.2
+          > SELECT MOD(2, 1.8);
+           0.2
+      
+  since: "1.0.0"
+
+- name: "*"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Multiply"
+  signatures:
+    - "expr1 * expr2"
+  usage: "expr1 * expr2 - Returns `expr1`*`expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 * 3;
+           6
+      
+  since: "1.0.0"
+
+- name: "+"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Add"
+  signatures:
+    - "expr1 + expr2"
+  usage: "expr1 + expr2 - Returns `expr1`+`expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 1 + 2;
+           3
+      
+  since: "1.0.0"
+
+- name: "-"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Subtract"
+  signatures:
+    - "expr1 - expr2"
+  usage: "expr1 - expr2 - Returns `expr1`-`expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 - 1;
+           1
+      
+  since: "1.0.0"
+
+- name: "/"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Divide"
+  signatures:
+    - "expr1 / expr2"
+  usage: "expr1 / expr2 - Returns `expr1`/`expr2`. It always performs floating point division."
+  examples: |2-
+    
+        Examples:
+          > SELECT 3 / 2;
+           1.5
+          > SELECT 2L / 2L;
+           1.0
+      
+  since: "1.0.0"
+
+- name: "abs"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Abs"
+  signatures:
+    - "abs(expr)"
+  usage: "abs(expr) - Returns the absolute value of the numeric or interval value."
+  examples: |2-
+    
+        Examples:
+          > SELECT abs(-1);
+           1
+          > SELECT abs(INTERVAL -'1-1' YEAR TO MONTH);
+           1-1
+      
+  since: "1.2.0"
+
+- name: "acos"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Acos"
+  signatures:
+    - "acos(expr)"
+  usage: |2-
+    
+        acos(expr) - Returns the inverse cosine (a.k.a. arc cosine) of `expr`, as if computed by
+          `java.lang.Math.acos`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT acos(1);
+           0.0
+          > SELECT acos(2);
+           NaN
+      
+  since: "1.4.0"
+
+- name: "acosh"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Acosh"
+  signatures:
+    - "acosh(expr)"
+  usage: |2-
+    
+        acosh(expr) - Returns inverse hyperbolic cosine of `expr`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT acosh(1);
+           0.0
+          > SELECT acosh(0);
+           NaN
+      
+  since: "3.0.0"
+
+- name: "asin"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Asin"
+  signatures:
+    - "asin(expr)"
+  usage: |2-
+    
+        asin(expr) - Returns the inverse sine (a.k.a. arc sine) the arc sin of `expr`,
+          as if computed by `java.lang.Math.asin`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT asin(0);
+           0.0
+          > SELECT asin(2);
+           NaN
+      
+  since: "1.4.0"
+
+- name: "asinh"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Asinh"
+  signatures:
+    - "asinh(expr)"
+  usage: |2-
+    
+        asinh(expr) - Returns inverse hyperbolic sine of `expr`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT asinh(0);
+           0.0
+      
+  since: "3.0.0"
+
+- name: "atan"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Atan"
+  signatures:
+    - "atan(expr)"
+  usage: |2-
+    
+        atan(expr) - Returns the inverse tangent (a.k.a. arc tangent) of `expr`, as if computed by
+          `java.lang.Math.atan`
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT atan(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "atan2"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Atan2"
+  signatures:
+    - "atan2(exprY, exprX)"
+  usage: |2-
+    
+        atan2(exprY, exprX) - Returns the angle in radians between the positive x-axis of a plane
+          and the point given by the coordinates (`exprX`, `exprY`), as if computed by
+          `java.lang.Math.atan2`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * exprY - coordinate on y-axis
+          * exprX - coordinate on x-axis
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT atan2(0, 0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "atanh"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Atanh"
+  signatures:
+    - "atanh(expr)"
+  usage: |2-
+    
+        atanh(expr) - Returns inverse hyperbolic tangent of `expr`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT atanh(0);
+           0.0
+          > SELECT atanh(2);
+           NaN
+      
+  since: "3.0.0"
+
+- name: "bin"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Bin"
+  signatures:
+    - "bin(expr)"
+  usage: "bin(expr) - Returns the string representation of the long value `expr` represented in binary."
+  examples: |2-
+    
+        Examples:
+          > SELECT bin(13);
+           1101
+          > SELECT bin(-13);
+           1111111111111111111111111111111111111111111111111111111111110011
+          > SELECT bin(13.3);
+           1101
+      
+  since: "1.5.0"
+
+- name: "bround"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BRound"
+  signatures:
+    - "bround(expr, d)"
+  usage: "bround(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_EVEN rounding mode."
+  examples: |2-
+    
+        Examples:
+          > SELECT bround(2.5, 0);
+           2
+          > SELECT bround(25, -1);
+           20
+      
+  since: "2.0.0"
+
+- name: "cbrt"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cbrt"
+  signatures:
+    - "cbrt(expr)"
+  usage: "cbrt(expr) - Returns the cube root of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT cbrt(27.0);
+           3.0
+      
+  since: "1.4.0"
+
+- name: "ceil"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CeilExpressionBuilder"
+  signatures:
+    - "ceil(expr[, scale])"
+  usage: "ceil(expr[, scale]) - Returns the smallest number after rounding up that is not smaller than `expr`. An optional `scale` parameter can be specified to control the rounding behavior."
+  examples: |2-
+    
+        Examples:
+          > SELECT ceil(-0.1);
+           0
+          > SELECT ceil(5);
+           5
+          > SELECT ceil(3.1411, 3);
+           3.142
+          > SELECT ceil(3.1411, -3);
+           1000
+      
+  since: "3.3.0"
+
+- name: "ceiling"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CeilExpressionBuilder"
+  signatures:
+    - "ceiling(expr[, scale])"
+  usage: "ceiling(expr[, scale]) - Returns the smallest number after rounding up that is not smaller than `expr`. An optional `scale` parameter can be specified to control the rounding behavior."
+  examples: |2-
+    
+        Examples:
+          > SELECT ceiling(-0.1);
+           0
+          > SELECT ceiling(5);
+           5
+          > SELECT ceiling(3.1411, 3);
+           3.142
+          > SELECT ceiling(3.1411, -3);
+           1000
+      
+  since: "3.3.0"
+
+- name: "conv"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Conv"
+  signatures:
+    - "conv(num, from_base, to_base)"
+  usage: "conv(num, from_base, to_base) - Convert `num` from `from_base` to `to_base`."
+  examples: |2-
+    
+        Examples:
+          > SELECT conv('100', 2, 10);
+           4
+          > SELECT conv(-10, 16, -10);
+           -16
+      
+  since: "1.5.0"
+
+- name: "cos"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cos"
+  signatures:
+    - "cos(expr)"
+  usage: |2-
+    
+        cos(expr) - Returns the cosine of `expr`, as if computed by
+          `java.lang.Math.cos`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT cos(0);
+           1.0
+      
+  since: "1.4.0"
+
+- name: "cosh"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cosh"
+  signatures:
+    - "cosh(expr)"
+  usage: |2-
+    
+          cosh(expr) - Returns the hyperbolic cosine of `expr`, as if computed by
+            `java.lang.Math.cosh`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - hyperbolic angle
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT cosh(0);
+           1.0
+      
+  since: "1.4.0"
+
+- name: "cot"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Cot"
+  signatures:
+    - "cot(expr)"
+  usage: |2-
+    
+        cot(expr) - Returns the cotangent of `expr`, as if computed by `1/java.lang.Math.tan`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT cot(1);
+           0.6420926159343306
+      
+  since: "2.3.0"
+
+- name: "csc"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Csc"
+  signatures:
+    - "csc(expr)"
+  usage: |2-
+    
+        csc(expr) - Returns the cosecant of `expr`, as if computed by `1/java.lang.Math.sin`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT csc(1);
+           1.1883951057781212
+      
+  since: "3.3.0"
+
+- name: "degrees"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToDegrees"
+  signatures:
+    - "degrees(expr)"
+  usage: "degrees(expr) - Converts radians to degrees."
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT degrees(3.141592653589793);
+           180.0
+      
+  since: "1.4.0"
+
+- name: "div"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.IntegralDivide"
+  signatures:
+    - "expr1 div expr2"
+  usage: "expr1 div expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long."
+  examples: |2-
+    
+        Examples:
+          > SELECT 3 div 2;
+           1
+          > SELECT INTERVAL '1-1' YEAR TO MONTH div INTERVAL '-1' MONTH;
+           -13
+      
+  since: "3.0.0"
+
+- name: "e"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.EulerNumber"
+  signatures:
+    - "e()"
+  usage: "e() - Returns Euler's number, e."
+  examples: |2-
+    
+        Examples:
+          > SELECT e();
+           2.718281828459045
+      
+  since: "1.5.0"
+
+- name: "exp"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Exp"
+  signatures:
+    - "exp(expr)"
+  usage: "exp(expr) - Returns e to the power of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT exp(0);
+           1.0
+      
+  since: "1.4.0"
+
+- name: "expm1"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Expm1"
+  signatures:
+    - "expm1(expr)"
+  usage: "expm1(expr) - Returns exp(`expr`) - 1."
+  examples: |2-
+    
+        Examples:
+          > SELECT expm1(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "factorial"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Factorial"
+  signatures:
+    - "factorial(expr)"
+  usage: "factorial(expr) - Returns the factorial of `expr`. `expr` is [0..20]. Otherwise, null."
+  examples: |2-
+    
+        Examples:
+          > SELECT factorial(5);
+           120
+      
+  since: "1.5.0"
+
+- name: "floor"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FloorExpressionBuilder"
+  signatures:
+    - "floor(expr[, scale])"
+  usage: " floor(expr[, scale]) - Returns the largest number after rounding down that is not greater than `expr`. An optional `scale` parameter can be specified to control the rounding behavior."
+  examples: |2-
+    
+        Examples:
+          > SELECT floor(-0.1);
+           -1
+          > SELECT floor(5);
+           5
+          > SELECT floor(3.1411, 3);
+           3.141
+          > SELECT floor(3.1411, -3);
+           0
+      
+  since: "3.3.0"
+
+- name: "greatest"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Greatest"
+  signatures:
+    - "greatest(expr, ...)"
+  usage: "greatest(expr, ...) - Returns the greatest value of all parameters, skipping null values."
+  examples: |2-
+    
+        Examples:
+          > SELECT greatest(10, 9, 2, 4, 3);
+           10
+      
+  since: "1.5.0"
+
+- name: "hex"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Hex"
+  signatures:
+    - "hex(expr)"
+  usage: "hex(expr) - Converts `expr` to hexadecimal."
+  examples: |2-
+    
+        Examples:
+          > SELECT hex(17);
+           11
+          > SELECT hex('Spark SQL');
+           537061726B2053514C
+      
+  since: "1.5.0"
+
+- name: "hypot"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Hypot"
+  signatures:
+    - "hypot(expr1, expr2)"
+  usage: "hypot(expr1, expr2) - Returns sqrt(`expr1`² + `expr2`²)."
+  examples: |2-
+    
+        Examples:
+          > SELECT hypot(3, 4);
+           5.0
+      
+  since: "1.4.0"
+
+- name: "least"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Least"
+  signatures:
+    - "least(expr, ...)"
+  usage: "least(expr, ...) - Returns the least value of all parameters, skipping null values."
+  examples: |2-
+    
+        Examples:
+          > SELECT least(10, 9, 2, 4, 3);
+           2
+      
+  since: "1.5.0"
+
+- name: "ln"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Log"
+  signatures:
+    - "ln(expr)"
+  usage: "ln(expr) - Returns the natural logarithm (base e) of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT ln(1);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "log"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Logarithm"
+  signatures:
+    - "log(base, expr)"
+  usage: "log(base, expr) - Returns the logarithm of `expr` with `base`."
+  examples: |2-
+    
+        Examples:
+          > SELECT log(10, 100);
+           2.0
+      
+  since: "1.5.0"
+
+- name: "log10"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Log10"
+  signatures:
+    - "log10(expr)"
+  usage: "log10(expr) - Returns the logarithm of `expr` with base 10."
+  examples: |2-
+    
+        Examples:
+          > SELECT log10(10);
+           1.0
+      
+  since: "1.4.0"
+
+- name: "log1p"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Log1p"
+  signatures:
+    - "log1p(expr)"
+  usage: "log1p(expr) - Returns log(1 + `expr`)."
+  examples: |2-
+    
+        Examples:
+          > SELECT log1p(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "log2"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Log2"
+  signatures:
+    - "log2(expr)"
+  usage: "log2(expr) - Returns the logarithm of `expr` with base 2."
+  examples: |2-
+    
+        Examples:
+          > SELECT log2(2);
+           1.0
+      
+  since: "1.4.0"
+
+- name: "mod"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Remainder"
+  signatures:
+    - "expr1 % expr2, or mod(expr1, expr2)"
+  usage: "expr1 % expr2, or mod(expr1, expr2) - Returns the remainder after `expr1`/`expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 % 1.8;
+           0.2
+          > SELECT MOD(2, 1.8);
+           0.2
+      
+  since: "2.3.0"
+
+- name: "negative"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnaryMinus"
+  signatures:
+    - "negative(expr)"
+  usage: "negative(expr) - Returns the negated value of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT negative(1);
+           -1
+      
+  since: "1.0.0"
+
+- name: "pi"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Pi"
+  signatures:
+    - "pi()"
+  usage: "pi() - Returns pi."
+  examples: |2-
+    
+        Examples:
+          > SELECT pi();
+           3.141592653589793
+      
+  since: "1.5.0"
+
+- name: "pmod"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Pmod"
+  signatures:
+    - "pmod(expr1, expr2)"
+  usage: "pmod(expr1, expr2) - Returns the positive value of `expr1` mod `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT pmod(10, 3);
+           1
+          > SELECT pmod(-10, 3);
+           2
+      
+  since: "1.5.0"
+
+- name: "positive"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnaryPositive"
+  signatures:
+    - "positive(expr)"
+  usage: "positive(expr) - Returns the value of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT positive(1);
+           1
+      
+  since: "1.5.0"
+
+- name: "pow"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Pow"
+  signatures:
+    - "pow(expr1, expr2)"
+  usage: "pow(expr1, expr2) - Raises `expr1` to the power of `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT pow(2, 3);
+           8.0
+      
+  since: "1.4.0"
+
+- name: "power"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Pow"
+  signatures:
+    - "power(expr1, expr2)"
+  usage: "power(expr1, expr2) - Raises `expr1` to the power of `expr2`."
+  examples: |2-
+    
+        Examples:
+          > SELECT power(2, 3);
+           8.0
+      
+  since: "1.4.0"
+
+- name: "radians"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToRadians"
+  signatures:
+    - "radians(expr)"
+  usage: "radians(expr) - Converts degrees to radians."
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in degrees
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT radians(180);
+           3.141592653589793
+      
+  since: "1.4.0"
+
+- name: "rand"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Rand"
+  signatures:
+    - "rand([seed])"
+  usage: "rand([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1)."
+  examples: |2-
+    
+        Examples:
+          > SELECT rand();
+           0.9629742951434543
+          > SELECT rand(0);
+           0.7604953758285915
+          > SELECT rand(null);
+           0.7604953758285915
+      
+  note: |2-
+    
+        The function is non-deterministic in general case.
+      
+  since: "1.5.0"
+
+- name: "randn"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Randn"
+  signatures:
+    - "randn([seed])"
+  usage: "randn([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution."
+  examples: |2-
+    
+        Examples:
+          > SELECT randn();
+           -0.3254147983080288
+          > SELECT randn(0);
+           1.6034991609278433
+          > SELECT randn(null);
+           1.6034991609278433
+      
+  note: |2-
+    
+        The function is non-deterministic in general case.
+      
+  since: "1.5.0"
+
+- name: "random"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Rand"
+  signatures:
+    - "random([seed])"
+  usage: "random([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1)."
+  examples: |2-
+    
+        Examples:
+          > SELECT random();
+           0.9629742951434543
+          > SELECT random(0);
+           0.7604953758285915
+          > SELECT random(null);
+           0.7604953758285915
+      
+  note: |2-
+    
+        The function is non-deterministic in general case.
+      
+  since: "3.0.0"
+
+- name: "random_poisson"
+  scope: "sail_extension"
+  signatures:
+    - "random_poisson(lambda), random_poisson(lambda, seed)"
+  usage: "random_poisson(lambda), random_poisson(lambda, seed) - Returns random values drawn from a Poisson distribution with rate lambda. If seed is provided, the generated sequence is deterministic."
+
+- name: "rint"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Rint"
+  signatures:
+    - "rint(expr)"
+  usage: "rint(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer."
+  examples: |2-
+    
+        Examples:
+          > SELECT rint(12.3456);
+           12.0
+      
+  since: "1.4.0"
+
+- name: "round"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Round"
+  signatures:
+    - "round(expr, d)"
+  usage: "round(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_UP rounding mode."
+  examples: |2-
+    
+        Examples:
+          > SELECT round(2.5, 0);
+           3
+      
+  since: "1.5.0"
+
+- name: "sec"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sec"
+  signatures:
+    - "sec(expr)"
+  usage: |2-
+    
+        sec(expr) - Returns the secant of `expr`, as if computed by `1/java.lang.Math.cos`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sec(0);
+           1.0
+      
+  since: "3.3.0"
+
+- name: "sign"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Signum"
+  signatures:
+    - "sign(expr)"
+  usage: "sign(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive."
+  examples: |2-
+    
+        Examples:
+          > SELECT sign(40);
+           1.0
+          > SELECT sign(INTERVAL -'100' YEAR);
+           -1.0
+      
+  since: "1.4.0"
+
+- name: "signum"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Signum"
+  signatures:
+    - "signum(expr)"
+  usage: "signum(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive."
+  examples: |2-
+    
+        Examples:
+          > SELECT signum(40);
+           1.0
+          > SELECT signum(INTERVAL -'100' YEAR);
+           -1.0
+      
+  since: "1.4.0"
+
+- name: "sin"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sin"
+  signatures:
+    - "sin(expr)"
+  usage: "sin(expr) - Returns the sine of `expr`, as if computed by `java.lang.Math.sin`."
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sin(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "sinh"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sinh"
+  signatures:
+    - "sinh(expr)"
+  usage: |2-
+    
+        sinh(expr) - Returns hyperbolic sine of `expr`, as if computed by `java.lang.Math.sinh`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - hyperbolic angle
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sinh(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "sqrt"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sqrt"
+  signatures:
+    - "sqrt(expr)"
+  usage: "sqrt(expr) - Returns the square root of `expr`."
+  examples: |2-
+    
+        Examples:
+          > SELECT sqrt(4);
+           2.0
+      
+  since: "1.1.1"
+
+- name: "tan"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Tan"
+  signatures:
+    - "tan(expr)"
+  usage: |2-
+    
+        tan(expr) - Returns the tangent of `expr`, as if computed by `java.lang.Math.tan`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - angle in radians
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT tan(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "tanh"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Tanh"
+  signatures:
+    - "tanh(expr)"
+  usage: |2-
+    
+        tanh(expr) - Returns the hyperbolic tangent of `expr`, as if computed by
+          `java.lang.Math.tanh`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - hyperbolic angle
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT tanh(0);
+           0.0
+      
+  since: "1.4.0"
+
+- name: "try_add"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryAdd"
+  signatures:
+    - "try_add(expr1, expr2)"
+  usage: "try_add(expr1, expr2) - Returns the sum of `expr1`and `expr2` and the result is null on overflow. The acceptable input types are the same with the `+` operator."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_add(1, 2);
+           3
+          > SELECT try_add(2147483647, 1);
+           NULL
+          > SELECT try_add(date'2021-01-01', 1);
+           2021-01-02
+          > SELECT try_add(date'2021-01-01', interval 1 year);
+           2022-01-01
+          > SELECT try_add(timestamp'2021-01-01 00:00:00', interval 1 day);
+           2021-01-02 00:00:00
+          > SELECT try_add(interval 1 year, interval 2 year);
+           3-0
+      
+  since: "3.2.0"
+
+- name: "try_divide"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryDivide"
+  signatures:
+    - "try_divide(dividend, divisor)"
+  usage: "try_divide(dividend, divisor) - Returns `dividend`/`divisor`. It always performs floating point division. Its result is always null if `expr2` is 0. `dividend` must be a numeric or an interval. `divisor` must be a numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_divide(3, 2);
+           1.5
+          > SELECT try_divide(2L, 2L);
+           1.0
+          > SELECT try_divide(1, 0);
+           NULL
+          > SELECT try_divide(interval 2 month, 2);
+           0-1
+          > SELECT try_divide(interval 2 month, 0);
+           NULL
+      
+  since: "3.2.0"
+
+- name: "try_mod"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryMod"
+  signatures:
+    - "try_mod(dividend, divisor)"
+  usage: "try_mod(dividend, divisor) - Returns the remainder after `expr1`/`expr2`. `dividend` must be a numeric. `divisor` must be a numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_mod(3, 2);
+           1
+          > SELECT try_mod(2L, 2L);
+           0
+          > SELECT try_mod(3.0, 2.0);
+           1.0
+          > SELECT try_mod(1, 0);
+           NULL
+      
+  since: "4.0.0"
+
+- name: "try_multiply"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryMultiply"
+  signatures:
+    - "try_multiply(expr1, expr2)"
+  usage: "try_multiply(expr1, expr2) - Returns `expr1`*`expr2` and the result is null on overflow. The acceptable input types are the same with the `*` operator."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_multiply(2, 3);
+           6
+          > SELECT try_multiply(-2147483648, 10);
+           NULL
+          > SELECT try_multiply(interval 2 year, 3);
+           6-0
+      
+  since: "3.3.0"
+
+- name: "try_subtract"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TrySubtract"
+  signatures:
+    - "try_subtract(expr1, expr2)"
+  usage: "try_subtract(expr1, expr2) - Returns `expr1`-`expr2` and the result is null on overflow. The acceptable input types are the same with the `-` operator."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_subtract(2, 1);
+           1
+          > SELECT try_subtract(-2147483648, 1);
+           NULL
+          > SELECT try_subtract(date'2021-01-02', 1);
+           2021-01-01
+          > SELECT try_subtract(date'2021-01-01', interval 1 year);
+           2020-01-01
+          > SELECT try_subtract(timestamp'2021-01-02 00:00:00', interval 1 day);
+           2021-01-01 00:00:00
+          > SELECT try_subtract(interval 2 year, interval 1 year);
+           1-0
+      
+  since: "3.3.0"
+
+- name: "unhex"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Unhex"
+  signatures:
+    - "unhex(expr)"
+  usage: "unhex(expr) - Converts hexadecimal `expr` to binary."
+  examples: |2-
+    
+        Examples:
+          > SELECT decode(unhex('537061726B2053514C'), 'UTF-8');
+           Spark SQL
+      
+  since: "1.5.0"
+
+- name: "uniform"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Uniform"
+  signatures:
+    - "uniform(min, max[, seed])"
+  usage: |2-
+    
+        uniform(min, max[, seed]) - Returns a random value with independent and identically
+          distributed (i.i.d.) values with the specified range of numbers. The random seed is optional.
+          The provided numbers specifying the minimum and maximum values of the range must be constant.
+          If both of these numbers are integers, then the result will also be an integer. Otherwise if
+          one or both of these are floating-point numbers, then the result will also be a floating-point
+          number.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT uniform(10, 20, 0) > 0 AS result;
+          true
+      
+  since: "4.0.0"
+
+- name: "width_bucket"
+  scope: "sql_builtin"
+  group: "math_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.WidthBucket"
+  signatures:
+    - "width_bucket(value, min_value, max_value, num_bucket)"
+  usage: |2-
+    
+        width_bucket(value, min_value, max_value, num_bucket) - Returns the bucket number to which
+          `value` would be assigned in an equiwidth histogram with `num_bucket` buckets,
+          in the range `min_value` to `max_value`."
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT width_bucket(5.3, 0.2, 10.6, 5);
+           3
+          > SELECT width_bucket(-2.1, 1.3, 3.4, 3);
+           0
+          > SELECT width_bucket(8.1, 0.0, 5.7, 4);
+           5
+          > SELECT width_bucket(-0.9, 5.2, 0.5, 2);
+           3
+          > SELECT width_bucket(INTERVAL '0' YEAR, INTERVAL '0' YEAR, INTERVAL '10' YEAR, 10);
+           1
+          > SELECT width_bucket(INTERVAL '1' YEAR, INTERVAL '0' YEAR, INTERVAL '10' YEAR, 10);
+           2
+          > SELECT width_bucket(INTERVAL '0' DAY, INTERVAL '0' DAY, INTERVAL '10' DAY, 10);
+           1
+          > SELECT width_bucket(INTERVAL '1' DAY, INTERVAL '0' DAY, INTERVAL '10' DAY, 10);
+           2
+      
+  since: "3.1.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/misc.yaml b/crates/sail-plan/data/functions/scalar/misc.yaml
new file mode 100644
index 000000000..00bd9f92a
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/misc.yaml
@@ -0,0 +1,1121 @@
+- name: "aes_decrypt"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.AesDecrypt"
+  signatures:
+    - "aes_decrypt(expr, key[, mode[, padding[, aad]]])"
+  usage: |2-
+    
+        aes_decrypt(expr, key[, mode[, padding[, aad]]]) - Returns a decrypted value of `expr` using AES in `mode` with `padding`.
+          Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS').
+          Optional additional authenticated data (AAD) is only supported for GCM. If provided for encryption, the identical AAD value must be provided for decryption.
+          The default mode is GCM.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - The binary value to decrypt.
+          * key - The passphrase to use to decrypt the data.
+          * mode - Specifies which block cipher mode should be used to decrypt messages.
+                   Valid modes: ECB, GCM, CBC.
+          * padding - Specifies how to pad messages whose length is not a multiple of the block size.
+                      Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC.
+          * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and
+                  must be provided for both encryption and decryption.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333');
+           Spark
+          > SELECT aes_decrypt(unhex('6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM');
+           Spark SQL
+          > SELECT aes_decrypt(unbase64('3lmwu+Mw0H3fi5NDvcu9lg=='), '1234567890abcdef', 'ECB', 'PKCS');
+           Spark SQL
+          > SELECT aes_decrypt(unbase64('2NYmDCjgXTbbxGA3/SnJEfFC/JQ7olk2VQWReIAAFKo='), '1234567890abcdef', 'CBC');
+           Apache Spark
+          > SELECT aes_decrypt(unbase64('AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg='), 'abcdefghijklmnop12345678ABCDEFGH', 'CBC', 'DEFAULT');
+           Spark
+          > SELECT aes_decrypt(unbase64('AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4'), 'abcdefghijklmnop12345678ABCDEFGH', 'GCM', 'DEFAULT', 'This is an AAD mixed into the input');
+           Spark
+      
+  since: "3.3.0"
+
+- name: "aes_encrypt"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.AesEncrypt"
+  signatures:
+    - "aes_encrypt(expr, key[, mode[, padding[, iv[, aad]]]])"
+  usage: |2-
+    
+        aes_encrypt(expr, key[, mode[, padding[, iv[, aad]]]]) - Returns an encrypted value of `expr` using AES in given `mode` with the specified `padding`.
+          Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS').
+          Optional initialization vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 bytes for GCM. If not provided, a random vector will be generated and prepended to the output.
+          Optional additional authenticated data (AAD) is only supported for GCM. If provided for encryption, the identical AAD value must be provided for decryption.
+          The default mode is GCM.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr - The binary value to encrypt.
+          * key - The passphrase to use to encrypt the data.
+          * mode - Specifies which block cipher mode should be used to encrypt messages.
+                   Valid modes: ECB, GCM, CBC.
+          * padding - Specifies how to pad messages whose length is not a multiple of the block size.
+                      Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC.
+          * iv - Optional initialization vector. Only supported for CBC and GCM modes.
+                 Valid values: None or ''. 16-byte array for CBC mode. 12-byte array for GCM mode.
+          * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and
+                  must be provided for both encryption and decryption.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT hex(aes_encrypt('Spark', '0000111122223333'));
+           83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94
+          > SELECT hex(aes_encrypt('Spark SQL', '0000111122223333', 'GCM'));
+           6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210
+          > SELECT base64(aes_encrypt('Spark SQL', '1234567890abcdef', 'ECB', 'PKCS'));
+           3lmwu+Mw0H3fi5NDvcu9lg==
+          > SELECT base64(aes_encrypt('Apache Spark', '1234567890abcdef', 'CBC', 'DEFAULT'));
+           2NYmDCjgXTbbxGA3/SnJEfFC/JQ7olk2VQWReIAAFKo=
+          > SELECT base64(aes_encrypt('Spark', 'abcdefghijklmnop12345678ABCDEFGH', 'CBC', 'DEFAULT', unhex('00000000000000000000000000000000')));
+           AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=
+          > SELECT base64(aes_encrypt('Spark', 'abcdefghijklmnop12345678ABCDEFGH', 'GCM', 'DEFAULT', unhex('000000000000000000000000'), 'This is an AAD mixed into the input'));
+           AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4
+      
+  since: "3.3.0"
+
+- name: "assert_true"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.AssertTrue"
+  signatures:
+    - "assert_true(expr [, message])"
+  usage: "assert_true(expr [, message]) - Throws an exception if `expr` is not true."
+  examples: |2-
+    
+        Examples:
+          > SELECT assert_true(0 < 1);
+           NULL
+      
+  since: "2.0.0"
+
+- name: "bitmap_bit_position"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitmapBitPosition"
+  signatures:
+    - "bitmap_bit_position(child)"
+  usage: "bitmap_bit_position(child) - Returns the bit position for the given input child expression."
+  examples: |2-
+    
+        Examples:
+          > SELECT bitmap_bit_position(1);
+           0
+          > SELECT bitmap_bit_position(123);
+           122
+      
+  since: "3.5.0"
+
+- name: "bitmap_bucket_number"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitmapBucketNumber"
+  signatures:
+    - "bitmap_bucket_number(child)"
+  usage: "bitmap_bucket_number(child) - Returns the bucket number for the given input child expression."
+  examples: |2-
+    
+        Examples:
+          > SELECT bitmap_bucket_number(123);
+           1
+          > SELECT bitmap_bucket_number(0);
+           0
+      
+  since: "3.5.0"
+
+- name: "bitmap_count"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitmapCount"
+  signatures:
+    - "bitmap_count(child)"
+  usage: "bitmap_count(child) - Returns the number of set bits in the child bitmap."
+  examples: |2-
+    
+        Examples:
+          > SELECT bitmap_count(X '1010');
+           2
+          > SELECT bitmap_count(X 'FFFF');
+           16
+          > SELECT bitmap_count(X '0');
+           0
+      
+  since: "3.5.0"
+
+- name: "current_catalog"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentCatalog"
+  signatures:
+    - "current_catalog()"
+  usage: "current_catalog() - Returns the current catalog."
+  examples: |2-
+    
+        Examples:
+          > SELECT current_catalog();
+           spark_catalog
+      
+  since: "3.1.0"
+
+- name: "current_database"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentDatabase"
+  signatures:
+    - "current_database()"
+  usage: "current_database() - Returns the current database."
+  examples: |2-
+    
+        Examples:
+          > SELECT current_database();
+           default
+      
+  since: "1.6.0"
+
+- name: "current_path"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentPath"
+  signatures:
+    - "current_path()"
+  usage: "current_path() - Returns the current SQL path (qualified schema names)."
+  examples: |2-
+    
+        Examples:
+          > SELECT current_path();
+           system.builtin,system.session,spark_catalog.default
+      
+  since: "4.2.0"
+
+- name: "current_schema"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentDatabase"
+  signatures:
+    - "current_schema()"
+  usage: "current_schema() - Returns the current database."
+  examples: |2-
+    
+        Examples:
+          > SELECT current_schema();
+           default
+      
+  since: "3.4.0"
+
+- name: "current_user"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentUser"
+  signatures:
+    - "current_user()"
+  usage: "current_user() - user name of current execution context."
+  examples: |2-
+    
+        Examples:
+          > SELECT current_user();
+           mockingjay
+      
+  since: "3.2.0"
+
+- name: "from_avro"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FromAvro"
+  signatures:
+    - "from_avro(child, jsonFormatSchema, options)"
+  usage: |2-
+    
+        from_avro(child, jsonFormatSchema, options) - Converts a binary Avro value into a Catalyst value.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT from_avro(s, '{"type": "record", "name": "struct", "fields": [{ "name": "u", "type": ["int","string"] }]}', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('u', NAMED_STRUCT('member0', member0, 'member1', member1)) AS s FROM VALUES (1, NULL), (NULL,  'a') tab(member0, member1));
+           [false]
+      
+  note: |2-
+    
+        The specified schema must match actual schema of the read data, otherwise the behavior
+        is undefined: it may fail or return arbitrary result.
+        To deserialize the data with a compatible and evolved schema, the expected Avro schema can be
+        set via the corresponding option.
+      
+  since: "4.0.0"
+
+- name: "from_protobuf"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FromProtobuf"
+  signatures:
+    - "from_protobuf(data, messageName, descFilePath, options)"
+  usage: |2-
+    
+        from_protobuf(data, messageName, descFilePath, options) - Converts a binary Protobuf value into a Catalyst value.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT from_protobuf(s, 'Person', '/path/to/descriptor.desc', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('name', name, 'id', id) AS s FROM VALUES ('John Doe', 1), (NULL,  2) tab(name, id));
+           [false]
+      
+  note: |2-
+    
+        The specified Protobuf schema must match actual schema of the read data, otherwise the behavior
+        is undefined: it may fail or return arbitrary result.
+        To deserialize the data with a compatible and evolved schema, the expected Protobuf schema can be
+        set via the corresponding option.
+      
+  since: "4.0.0"
+
+- name: "hll_sketch_estimate"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.HllSketchEstimate"
+  signatures:
+    - "hll_sketch_estimate(expr)"
+  usage: |2-
+    
+        hll_sketch_estimate(expr) - Returns the estimated number of unique values given the binary representation
+        of a Datasketches HllSketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT hll_sketch_estimate(hll_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);
+           3
+      
+  since: "3.5.0"
+
+- name: "hll_union"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.HllUnion"
+  signatures:
+    - "hll_union(first, second, allowDifferentLgConfigK)"
+  usage: |2-
+    
+        hll_union(first, second, allowDifferentLgConfigK) - Merges two binary representations of
+        Datasketches HllSketch objects, using a Datasketches Union object. Set
+        allowDifferentLgConfigK to true to allow unions of sketches with different
+        lgConfigK values (defaults to false). 
+  examples: |2-
+    
+        Examples:
+          > SELECT hll_sketch_estimate(hll_union(hll_sketch_agg(col1), hll_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);
+           6
+      
+  since: "3.5.0"
+
+- name: "input_file_block_length"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.InputFileBlockLength"
+  signatures:
+    - "input_file_block_length()"
+  usage: "input_file_block_length() - Returns the length of the block being read, or -1 if not available."
+  examples: |2-
+    
+        Examples:
+          > SELECT input_file_block_length();
+           -1
+      
+  since: "2.2.0"
+
+- name: "input_file_block_start"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.InputFileBlockStart"
+  signatures:
+    - "input_file_block_start()"
+  usage: "input_file_block_start() - Returns the start offset of the block being read, or -1 if not available."
+  examples: |2-
+    
+        Examples:
+          > SELECT input_file_block_start();
+           -1
+      
+  since: "2.2.0"
+
+- name: "input_file_name"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.InputFileName"
+  signatures:
+    - "input_file_name()"
+  usage: "input_file_name() - Returns the name of the file being read, or empty string if not available."
+  examples: |2-
+    
+        Examples:
+          > SELECT input_file_name();
+    
+      
+  since: "1.5.0"
+
+- name: "java_method"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection"
+  signatures:
+    - "java_method(class, method[, arg1[, arg2 ..]])"
+  usage: "java_method(class, method[, arg1[, arg2 ..]]) - Calls a method with reflection."
+  examples: |2-
+    
+        Examples:
+          > SELECT java_method('java.util.UUID', 'randomUUID');
+           c33fb387-8500-4bfa-81d2-6e0e3e930df2
+          > SELECT java_method('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2');
+           a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
+      
+  since: "2.0.0"
+
+- name: "monotonically_increasing_id"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID"
+  signatures:
+    - "monotonically_increasing_id()"
+  usage: |2-
+    
+        monotonically_increasing_id() - Returns monotonically increasing 64-bit integers. The generated ID is guaranteed
+          to be monotonically increasing and unique, but not consecutive. The current implementation
+          puts the partition ID in the upper 31 bits, and the lower 33 bits represent the record number
+          within each partition. The assumption is that the data frame has less than 1 billion
+          partitions, and each partition has less than 8 billion records.
+          The function is non-deterministic because its result depends on partition IDs.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT monotonically_increasing_id();
+           0
+      
+  since: "1.4.0"
+
+- name: "raise_error"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RaiseErrorExpressionBuilder"
+  signatures:
+    - "raise_error( expr )"
+  usage: "raise_error( expr ) - Throws a USER_RAISED_EXCEPTION with `expr` as message."
+  examples: |2-
+    
+        Examples:
+          > SELECT raise_error('custom error message');
+           [USER_RAISED_EXCEPTION] custom error message
+      
+  since: "3.1.0"
+
+- name: "reflect"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection"
+  signatures:
+    - "reflect(class, method[, arg1[, arg2 ..]])"
+  usage: "reflect(class, method[, arg1[, arg2 ..]]) - Calls a method with reflection."
+  examples: |2-
+    
+        Examples:
+          > SELECT reflect('java.util.UUID', 'randomUUID');
+           c33fb387-8500-4bfa-81d2-6e0e3e930df2
+          > SELECT reflect('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2');
+           a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
+      
+  since: "2.0.0"
+
+- name: "schema_of_avro"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SchemaOfAvro"
+  signatures:
+    - "schema_of_avro(jsonFormatSchema, options)"
+  usage: |2-
+    
+        schema_of_avro(jsonFormatSchema, options) - Returns schema in the DDL format of the avro schema in JSON string format.
+        
+  examples: |2-
+    
+        Examples:
+          > SELECT schema_of_avro('{"type": "record", "name": "struct", "fields": [{"name": "u", "type": ["int", "string"]}]}', map());
+           STRUCT NOT NULL>
+      
+  since: "4.0.0"
+
+- name: "session_user"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentUser"
+  signatures:
+    - "session_user()"
+  usage: "session_user() - user name of current execution context."
+  examples: |2-
+    
+        Examples:
+          > SELECT session_user();
+           mockingjay
+      
+  since: "4.0.0"
+
+- name: "spark_partition_id"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SparkPartitionID"
+  signatures:
+    - "spark_partition_id()"
+  usage: "spark_partition_id() - Returns the current partition id."
+  examples: |2-
+    
+        Examples:
+          > SELECT spark_partition_id();
+           0
+      
+  since: "1.4.0"
+
+- name: "theta_difference"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ThetaDifference"
+  signatures:
+    - "theta_difference(first, second)"
+  usage: |2-
+    
+        theta_difference(first, second) - Subtracts two binary representations of
+        Datasketches ThetaSketch objects from two input columns using a
+        ThetaSketch AnotB object. 
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_difference(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);
+           2
+      
+  since: "4.1.0"
+
+- name: "theta_intersection"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ThetaIntersection"
+  signatures:
+    - "theta_intersection(first, second)"
+  usage: |2-
+    
+        theta_intersection(first, second) - Intersects two binary representations of
+        Datasketches ThetaSketch objects from two input columns using a
+        ThetaSketch Intersect object. 
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);
+           2
+      
+  since: "4.1.0"
+
+- name: "theta_sketch_estimate"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ThetaSketchEstimate"
+  signatures:
+    - "theta_sketch_estimate(expr)"
+  usage: |2-
+    
+        theta_sketch_estimate(expr) - Returns the estimated number of unique values
+        given the binary representation of a Datasketches ThetaSketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);
+           3
+      
+  since: "4.1.0"
+
+- name: "theta_union"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ThetaUnion"
+  signatures:
+    - "theta_union(first, second, lgNomEntries)"
+  usage: |2-
+    
+        theta_union(first, second, lgNomEntries) - Merges two binary representations of
+        Datasketches ThetaSketch objects using a ThetaSketch Union object. Users can set
+        lgNomEntries to a value between 4 and 26 to find the union of sketches with different
+        union buffer size values (defaults to 12). 
+  examples: |2-
+    
+        Examples:
+          > SELECT theta_sketch_estimate(theta_union(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);
+           6
+      
+  since: "4.1.0"
+
+- name: "to_avro"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToAvro"
+  signatures:
+    - "to_avro(child[, jsonFormatSchema])"
+  usage: |2-
+    
+        to_avro(child[, jsonFormatSchema]) - Converts a Catalyst binary input value into its
+          corresponding Avro format result.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_avro(s, '{"type": "record", "name": "struct", "fields": [{ "name": "u", "type": ["int","string"] }]}') IS NULL FROM (SELECT NULL AS s);
+           [true]
+          > SELECT to_avro(s) IS NULL FROM (SELECT NULL AS s);
+           [true]
+      
+  since: "4.0.0"
+
+- name: "to_protobuf"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToProtobuf"
+  signatures:
+    - "to_protobuf(child, messageName, descFilePath, options)"
+  usage: |2-
+    
+        to_protobuf(child, messageName, descFilePath, options) - Converts a Catalyst binary input value into its corresponding
+          Protobuf format result.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_protobuf(s, 'Person', '/path/to/descriptor.desc', map('emitDefaultValues', 'true')) IS NULL FROM (SELECT NULL AS s);
+           [true]
+      
+  since: "4.0.0"
+
+- name: "try_aes_decrypt"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryAesDecrypt"
+  signatures:
+    - "try_aes_decrypt(expr, key[, mode[, padding[, aad]]])"
+  usage: "try_aes_decrypt(expr, key[, mode[, padding[, aad]]]) - This is a special version of `aes_decrypt` that performs the same operation, but returns a NULL value instead of raising an error if the decryption cannot be performed."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_aes_decrypt(unhex('6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM');
+           Spark SQL
+          > SELECT try_aes_decrypt(unhex('----------468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM');
+           NULL
+      
+  since: "3.5.0"
+
+- name: "try_aes_encrypt"
+  scope: "sail_extension"
+  signatures:
+    - "try_aes_encrypt(expr, key[, mode[, padding[, iv[, aad]]]])"
+  usage: "try_aes_encrypt(expr, key[, mode[, padding[, iv[, aad]]]]) - This is a special version of aes_encrypt that performs the same operation, but returns a NULL value instead of raising an error if encryption cannot be performed."
+
+- name: "try_reflect"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryReflect"
+  signatures:
+    - "try_reflect(class, method[, arg1[, arg2 ..]])"
+  usage: "try_reflect(class, method[, arg1[, arg2 ..]]) - This is a special version of `reflect` that performs the same operation, but returns a NULL value instead of raising an error if the invoke method thrown exception."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_reflect('java.util.UUID', 'randomUUID');
+           c33fb387-8500-4bfa-81d2-6e0e3e930df2
+          > SELECT try_reflect('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2');
+           a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
+          > SELECT try_reflect('java.net.URLDecoder', 'decode', '%');
+           NULL
+      
+  since: "4.0.0"
+
+- name: "tuple_difference_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleDifferenceDouble"
+  signatures:
+    - "tuple_difference_double(tupleSketch1, tupleSketch2)"
+  usage: |2-
+    
+        tuple_difference_double(tupleSketch1, tupleSketch2) - Subtracts two binary representations of Datasketches
+        TupleSketch objects with double summary data type using a TupleSketch AnotB object.
+        Returns elements in the first sketch that are not in the second sketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_difference_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (5, 5.0D, 4, 4.0D), (1, 1.0D, 4, 4.0D), (2, 2.0D, 5, 5.0D), (3, 3.0D, 1, 1.0D) tab(col1, val1, col2, val2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_difference_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleDifferenceInteger"
+  signatures:
+    - "tuple_difference_integer(tupleSketch1, tupleSketch2)"
+  usage: |2-
+    
+        tuple_difference_integer(tupleSketch1, tupleSketch2) - Subtracts two binary representations of Datasketches
+        TupleSketch objects with integer summary data type using a TupleSketch AnotB object.
+        Returns elements in the first sketch that are not in the second sketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_difference_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (5, 5, 4, 4), (1, 1, 4, 4), (2, 2, 5, 5), (3, 3, 1, 1) tab(col1, val1, col2, val2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_difference_theta_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleDifferenceThetaDouble"
+  signatures:
+    - "tuple_difference_theta_double(tupleSketch, thetaSketch)"
+  usage: |2-
+    
+        tuple_difference_theta_double(tupleSketch, thetaSketch) - Subtracts the binary representation of a
+        Datasketches ThetaSketch from a TupleSketch with double summary data type using a TupleSketch
+        AnotB object. Returns elements in the TupleSketch that are not in the ThetaSketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_difference_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (5, 5.0D, 4), (1, 1.0D, 4), (2, 2.0D, 5), (3, 3.0D, 1) tab(col1, val1, col2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_difference_theta_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleDifferenceThetaInteger"
+  signatures:
+    - "tuple_difference_theta_integer(tupleSketch, thetaSketch)"
+  usage: |2-
+    
+        tuple_difference_theta_integer(tupleSketch, thetaSketch) - Subtracts the binary representation of a
+        Datasketches ThetaSketch from a TupleSketch with integer summary data type using a TupleSketch
+        AnotB object. Returns elements in the TupleSketch that are not in the ThetaSketch. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_difference_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1) tab(col1, val1, col2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_intersection_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleIntersectionDouble"
+  signatures:
+    - "tuple_intersection_double(tupleSketch1, tupleSketch2, mode)"
+  usage: |2-
+    
+        tuple_intersection_double(tupleSketch1, tupleSketch2, mode) - Intersects two binary representations of Datasketches
+        TupleSketch objects with double summary data type using a TupleSketch Intersection object.
+        Users can set mode to 'sum', 'min', 'max', or 'alwaysone' (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_intersection_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (1, 1.0D, 1, 4.0D), (2, 2.0D, 2, 5.0D), (3, 3.0D, 4, 6.0D) tab(col1, val1, col2, val2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_intersection_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleIntersectionInteger"
+  signatures:
+    - "tuple_intersection_integer(tupleSketch1, tupleSketch2, mode)"
+  usage: |2-
+    
+        tuple_intersection_integer(tupleSketch1, tupleSketch2, mode) - Intersects two binary representations of Datasketches
+        TupleSketch objects with integer summary data type using a TupleSketch Intersection object.
+        Users can set mode to 'sum', 'min', 'max', or 'alwaysone' (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_intersection_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (1, 1, 1, 4), (2, 2, 2, 5), (3, 3, 4, 6) tab(col1, val1, col2, val2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_intersection_theta_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleIntersectionThetaDouble"
+  signatures:
+    - "tuple_intersection_theta_double(tupleSketch, thetaSketch, mode)"
+  usage: |2-
+    
+        tuple_intersection_theta_double(tupleSketch, thetaSketch, mode) - Intersects the binary representation of a
+        Datasketches TupleSketch with double summary data type with the binary representation of a
+        Datasketches ThetaSketch using a TupleSketch Intersection object. The ThetaSketch entries are
+        assigned a default double summary value based on the mode: 0.0 for 'sum' mode, +Infinity for
+        'min' mode, -Infinity for 'max' mode, or 1.0 for 'alwaysone' mode. Users can set mode to 'sum',
+        'min', 'max', or 'alwaysone' (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_intersection_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1.0D, 1), (2, 2.0D, 2), (3, 3.0D, 4) tab(col1, val1, col2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_intersection_theta_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleIntersectionThetaInteger"
+  signatures:
+    - "tuple_intersection_theta_integer(tupleSketch, thetaSketch, mode)"
+  usage: |2-
+    
+        tuple_intersection_theta_integer(tupleSketch, thetaSketch, mode) - Intersects the binary representation of a
+        Datasketches TupleSketch with integer summary data type with the binary representation of a
+        Datasketches ThetaSketch using a TupleSketch Intersection object. The ThetaSketch entries are
+        assigned a default integer summary value based on the mode: 0 for 'sum' mode, Integer.MAX_VALUE
+        for 'min' mode, Integer.MIN_VALUE for 'max' mode, or 1 for 'alwaysone' mode. Users can set mode
+        to 'sum', 'min', 'max', or 'alwaysone' (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_intersection_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1, 1), (2, 2, 2), (3, 3, 4) tab(col1, val1, col2);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_estimate_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleSketchEstimateDouble"
+  signatures:
+    - "tuple_sketch_estimate_double(child)"
+  usage: |2-
+    
+        tuple_sketch_estimate_double(child) - Returns the estimated number of unique values
+        given the binary representation of a Datasketches TupleSketch. The sketch's
+        summary type must be a double. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (1, 2.0D), (2, 3.0D) tab(key, summary);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_estimate_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleSketchEstimateInteger"
+  signatures:
+    - "tuple_sketch_estimate_integer(child)"
+  usage: |2-
+    
+        tuple_sketch_estimate_integer(child) - Returns the estimated number of unique values
+        given the binary representation of a Datasketches TupleSketch. The sketch's
+        summary type must be an integer. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (1, 2), (2, 3) tab(key, summary);
+           2.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_summary_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleSketchSummaryDouble"
+  signatures:
+    - "tuple_sketch_summary_double(child, mode)"
+  usage: |2-
+    
+        tuple_sketch_summary_double(child, mode) - Aggregates the summary values from a double summary type
+        Datasketches TupleSketch. The mode can be 'sum', 'min', 'max', or 'alwaysone'
+        (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_summary_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (1, 2.0D), (2, 3.0D) tab(key, summary);
+           6.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_summary_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleSketchSummaryInteger"
+  signatures:
+    - "tuple_sketch_summary_integer(child, mode)"
+  usage: |2-
+    
+        tuple_sketch_summary_integer(child, mode) - Aggregates the summary values from a integer summary type
+        Datasketches TupleSketch. The mode can be 'sum', 'min', 'max', or 'alwaysone'
+        (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (1, 2), (2, 3) tab(key, summary);
+           6
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_theta_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleSketchThetaDouble"
+  signatures:
+    - "tuple_sketch_theta_double(child)"
+  usage: |2-
+    
+        tuple_sketch_theta_double(child) - Returns the theta value (sampling rate) from a Datasketches TupleSketch.
+        The theta value represents the effective sampling rate of the sketch, between 0.0 and 1.0.
+        The sketch's summary type must be a double. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_theta_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (2, 2.0D), (3, 3.0D) tab(key, summary);
+           1.0
+      
+  since: "4.2.0"
+
+- name: "tuple_sketch_theta_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleSketchThetaInteger"
+  signatures:
+    - "tuple_sketch_theta_integer(child)"
+  usage: |2-
+    
+        tuple_sketch_theta_integer(child) - Returns the theta value (sampling rate) from a Datasketches TupleSketch.
+        The theta value represents the effective sampling rate of the sketch, between 0.0 and 1.0.
+        The sketch's summary type must be an integer. 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary);
+           1.0
+      
+  since: "4.2.0"
+
+- name: "tuple_union_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleUnionDoubleExpressionBuilder"
+  signatures:
+    - "tuple_union_double(tupleSketch1, tupleSketch2, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_union_double(tupleSketch1, tupleSketch2, lgNomEntries, mode) - Merges two binary representations of Datasketches
+        TupleSketch objects with double summary data type using a TupleSketch Union object. Users can
+        set lgNomEntries to a value between 4 and 26 (defaults to 12) and mode to 'sum', 'min', 'max',
+        or 'alwaysone' (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_union_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (1, 1.0D, 4, 4.0D), (2, 2.0D, 5, 5.0D), (3, 3.0D, 6, 6.0D) tab(col1, val1, col2, val2);
+           6.0
+      
+  since: "4.2.0"
+
+- name: "tuple_union_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleUnionIntegerExpressionBuilder"
+  signatures:
+    - "tuple_union_integer(tupleSketch1, tupleSketch2, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_union_integer(tupleSketch1, tupleSketch2, lgNomEntries, mode) - Merges two binary representations of Datasketches
+        TupleSketch objects with integer summary data type using a TupleSketch Union object. Users can
+        set lgNomEntries to a value between 4 and 26 (defaults to 12) and mode to 'sum', 'min', 'max',
+        or 'alwaysone' (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_union_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (1, 1, 4, 4), (2, 2, 5, 5), (3, 3, 6, 6) tab(col1, val1, col2, val2);
+           6.0
+      
+  since: "4.2.0"
+
+- name: "tuple_union_theta_double"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleUnionThetaDoubleExpressionBuilder"
+  signatures:
+    - "tuple_union_theta_double(tupleSketch, thetaSketch, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_union_theta_double(tupleSketch, thetaSketch, lgNomEntries, mode) - Merges the binary representation of a
+        Datasketches TupleSketch with double summary data type with the binary representation of a
+        Datasketches ThetaSketch using a TupleSketch Union object. The ThetaSketch entries are assigned
+        a default double summary value based on the mode: 0.0 for 'sum' mode, +Infinity for 'min' mode,
+        -Infinity for 'max' mode, or 1.0 for 'alwaysone' mode. Users can set lgNomEntries to a value
+        between 4 and 26 (defaults to 12) and mode to 'sum', 'min', 'max', or 'alwaysone' (defaults to
+        'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_double(tuple_union_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1.0D, 4), (2, 2.0D, 5), (3, 3.0D, 6) tab(col1, val1, col2);
+           6.0
+      
+  since: "4.2.0"
+
+- name: "tuple_union_theta_integer"
+  scope: "sql_builtin"
+  group: "sketch_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TupleUnionThetaIntegerExpressionBuilder"
+  signatures:
+    - "tuple_union_theta_integer(tupleSketch, thetaSketch, lgNomEntries, mode)"
+  usage: |2-
+    
+        tuple_union_theta_integer(tupleSketch, thetaSketch, lgNomEntries, mode) - Merges the binary representation of a
+        Datasketches TupleSketch with integer summary data type with the binary representation of a
+        Datasketches ThetaSketch using a TupleSketch Union object. The ThetaSketch entries are assigned
+        a default integer summary value based on the mode: 0 for 'sum' mode, Integer.MAX_VALUE for 'min'
+        mode, Integer.MIN_VALUE for 'max' mode, or 1 for 'alwaysone' mode. Users can set lgNomEntries to
+        a value between 4 and 26 (defaults to 12) and mode to 'sum', 'min', 'max', or 'alwaysone'
+        (defaults to 'sum'). 
+  examples: |2-
+    
+        Examples:
+          > SELECT tuple_sketch_estimate_integer(tuple_union_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1, 4), (2, 2, 5), (3, 3, 6) tab(col1, val1, col2);
+           6.0
+      
+  since: "4.2.0"
+
+- name: "typeof"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TypeOf"
+  signatures:
+    - "typeof(expr)"
+  usage: "typeof(expr) - Return DDL-formatted type string for the data type of the input."
+  examples: |2-
+    
+        Examples:
+          > SELECT typeof(1);
+           int
+          > SELECT typeof(array(1));
+           array
+      
+  since: "3.0.0"
+
+- name: "user"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CurrentUser"
+  signatures:
+    - "user()"
+  usage: "user() - user name of current execution context."
+  examples: |2-
+    
+        Examples:
+          > SELECT user();
+           mockingjay
+      
+  since: "3.4.0"
+
+- name: "uuid"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Uuid"
+  signatures:
+    - "uuid()"
+  usage: "uuid() - Returns an universally unique identifier (UUID) string. The value is returned as a canonical UUID 36-character string."
+  examples: |2-
+    
+        Examples:
+          > SELECT uuid();
+           46707d92-02f4-4817-8116-a4c3b23e6266
+      
+  note: |2-
+    
+        The function is non-deterministic.
+      
+  since: "2.3.0"
+
+- name: "version"
+  scope: "sql_builtin"
+  group: "misc_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SparkVersion"
+  signatures:
+    - "version()"
+  usage: "version() - Returns the Spark version. The string contains 2 fields, the first being a release version and the second being a git revision."
+  examples: |2-
+    
+        Examples:
+          > SELECT version();
+           3.1.0 a6d6ea3efedbad14d99c24143834cd4e2e52fb40
+      
+  since: "3.0.0"
diff --git a/crates/sail-plan/data/functions/scalar/predicate.yaml b/crates/sail-plan/data/functions/scalar/predicate.yaml
new file mode 100644
index 000000000..80986d940
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/predicate.yaml
@@ -0,0 +1,728 @@
+- name: "!"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Not"
+  signatures:
+    - "! expr"
+  usage: "! expr - Logical not."
+  examples: |2-
+    
+        Examples:
+          > SELECT ! true;
+           false
+          > SELECT ! false;
+           true
+          > SELECT ! NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "!="
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  signatures:
+    - "expr1 != expr2"
+  usage: "expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to
+                           a common type, and must be a type that can be used in equality comparison.
+                           Map type is not supported. For complex types such array/struct,
+                           the data types of fields must be orderable.
+  examples: |2-
+    
+        Examples:
+          > SELECT 1 != 2;
+           true
+          > SELECT 1 != '2';
+           true
+          > SELECT true != NULL;
+           NULL
+          > SELECT NULL != NULL;
+           NULL
+  since: "1.0.0"
+
+- name: "<"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.LessThan"
+  signatures:
+    - "expr1 < expr2"
+  usage: "expr1 < expr2 - Returns true if `expr1` is less than `expr2`."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be ordered. For example, map type is not orderable, so it
+              is not supported. For complex types such array/struct, the data types of fields must
+              be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 1 < 2;
+           true
+          > SELECT 1.1 < '1';
+           false
+          > SELECT to_date('2009-07-30 04:17:52') < to_date('2009-07-30 04:17:52');
+           false
+          > SELECT to_date('2009-07-30 04:17:52') < to_date('2009-08-01 04:17:52');
+           true
+          > SELECT 1 < NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "<="
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.LessThanOrEqual"
+  signatures:
+    - "expr1 <= expr2"
+  usage: "expr1 <= expr2 - Returns true if `expr1` is less than or equal to `expr2`."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be ordered. For example, map type is not orderable, so it
+              is not supported. For complex types such array/struct, the data types of fields must
+              be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 <= 2;
+           true
+          > SELECT 1.0 <= '1';
+           true
+          > SELECT to_date('2009-07-30 04:17:52') <= to_date('2009-07-30 04:17:52');
+           true
+          > SELECT to_date('2009-07-30 04:17:52') <= to_date('2009-08-01 04:17:52');
+           true
+          > SELECT 1 <= NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "<=>"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.EqualNullSafe"
+  signatures:
+    - "expr1 <=> expr2"
+  usage: |2-
+    
+        expr1 <=> expr2 - Returns same result as the EQUAL(=) operator for non-null operands,
+          but returns true if both are null, false if one of the them is null.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be used in equality comparison. Map type is not supported.
+              For complex types such array/struct, the data types of fields must be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 <=> 2;
+           true
+          > SELECT 1 <=> '1';
+           true
+          > SELECT true <=> NULL;
+           false
+          > SELECT NULL <=> NULL;
+           true
+      
+  since: "1.1.0"
+
+- name: "<>"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  signatures:
+    - "expr1 != expr2"
+  usage: "expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to
+                           a common type, and must be a type that can be used in equality comparison.
+                           Map type is not supported. For complex types such array/struct,
+                           the data types of fields must be orderable.
+  examples: |2-
+    
+        Examples:
+          > SELECT 1 != 2;
+           true
+          > SELECT 1 != '2';
+           true
+          > SELECT true != NULL;
+           NULL
+          > SELECT NULL != NULL;
+           NULL
+  since: "1.0.0"
+
+- name: "="
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.EqualTo"
+  signatures:
+    - "expr1 = expr2"
+  usage: "expr1 = expr2 - Returns true if `expr1` equals `expr2`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be used in equality comparison. Map type is not supported.
+              For complex types such array/struct, the data types of fields must be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 = 2;
+           true
+          > SELECT 1 = '1';
+           true
+          > SELECT true = NULL;
+           NULL
+          > SELECT NULL = NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "=="
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.EqualTo"
+  signatures:
+    - "expr1 == expr2"
+  usage: "expr1 == expr2 - Returns true if `expr1` equals `expr2`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be used in equality comparison. Map type is not supported.
+              For complex types such array/struct, the data types of fields must be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 == 2;
+           true
+          > SELECT 1 == '1';
+           true
+          > SELECT true == NULL;
+           NULL
+          > SELECT NULL == NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: ">"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.GreaterThan"
+  signatures:
+    - "expr1 > expr2"
+  usage: "expr1 > expr2 - Returns true if `expr1` is greater than `expr2`."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be ordered. For example, map type is not orderable, so it
+              is not supported. For complex types such array/struct, the data types of fields must
+              be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 > 1;
+           true
+          > SELECT 2 > 1.1;
+           true
+          > SELECT to_date('2009-07-30 04:17:52') > to_date('2009-07-30 04:17:52');
+           false
+          > SELECT to_date('2009-07-30 04:17:52') > to_date('2009-08-01 04:17:52');
+           false
+          > SELECT 1 > NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: ">="
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual"
+  signatures:
+    - "expr1 >= expr2"
+  usage: "expr1 >= expr2 - Returns true if `expr1` is greater than or equal to `expr2`."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be ordered. For example, map type is not orderable, so it
+              is not supported. For complex types such array/struct, the data types of fields must
+              be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 2 >= 1;
+           true
+          > SELECT 2.0 >= '2.1';
+           false
+          > SELECT to_date('2009-07-30 04:17:52') >= to_date('2009-07-30 04:17:52');
+           true
+          > SELECT to_date('2009-07-30 04:17:52') >= to_date('2009-08-01 04:17:52');
+           false
+          > SELECT 1 >= NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "and"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.And"
+  signatures:
+    - "expr1 and expr2"
+  usage: "expr1 and expr2 - Logical AND."
+  examples: |2-
+    
+        Examples:
+          > SELECT true and true;
+           true
+          > SELECT true and false;
+           false
+          > SELECT true and NULL;
+           NULL
+          > SELECT false and NULL;
+           false
+      
+  since: "1.0.0"
+
+- name: "between"
+  scope: "sql_builtin"
+  group: "conditional_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Between"
+  signatures:
+    - "input [NOT] between lower AND upper"
+  usage: "input [NOT] between lower AND upper - evaluate if `input` is [not] in between `lower` and `upper`"
+  arguments: |2-
+    
+        Arguments:
+          * input - An expression that is being compared with lower and upper bound.
+          * lower - Lower bound of the between check.
+          * upper - Upper bound of the between check.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 0.5 between 0.1 AND 1.0;
+            true
+      
+  since: "1.0.0"
+
+- name: "equal_null"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.EqualNull"
+  signatures:
+    - "equal_null(expr1, expr2)"
+  usage: |2-
+    
+        equal_null(expr1, expr2) - Returns same result as the EQUAL(=) operator for non-null operands,
+          but returns true if both are null, false if one of the them is null.
+      
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2 - the two expressions must be same type or can be casted to a common type,
+              and must be a type that can be used in equality comparison. Map type is not supported.
+              For complex types such array/struct, the data types of fields must be orderable.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT equal_null(3, 3);
+           true
+          > SELECT equal_null(1, '11');
+           false
+          > SELECT equal_null(true, NULL);
+           false
+          > SELECT equal_null(NULL, 'abc');
+           false
+          > SELECT equal_null(NULL, NULL);
+           true
+      
+  since: "3.4.0"
+
+- name: "ilike"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ILike"
+  signatures:
+    - "str ilike pattern[ ESCAPE escape]"
+  usage: "str ilike pattern[ ESCAPE escape] - Returns true if str matches `pattern` with `escape` case-insensitively, null if any arguments are null, false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * pattern - a string expression. The pattern is a string which is matched literally and
+              case-insensitively, with exception to the following special symbols:


+              _ matches any one character in the input (similar to . in posix regular expressions)


+              % matches zero or more characters in the input (similar to .* in posix regular
+              expressions)


+              Since Spark 2.0, string literals are unescaped in our SQL parser, see the unescaping
+              rules at String Literal.
+              For example, in order to match "\abc", the pattern should be "\\abc".


+              When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back
+              to Spark 1.6 behavior regarding string literal parsing. For example, if the config is
+              enabled, the pattern to match "\abc" should be "\abc".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+          * escape - an character added since Spark 3.0. The default escape character is the '\'.
+              If an escape character precedes a special symbol or another escape character, the
+              following character is matched literally. It is invalid to escape any other character.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT ilike('Spark', '_Park');
+          true
+          > SELECT '\\abc' AS S, S ilike r'\\abc', S ilike '\\\\abc';
+          \abc	true	true
+          > SET spark.sql.parser.escapedStringLiterals=true;
+          spark.sql.parser.escapedStringLiterals	true
+          > SELECT '%SystemDrive%\Users\John' ilike '\%SystemDrive\%\\users%';
+          true
+          > SET spark.sql.parser.escapedStringLiterals=false;
+          spark.sql.parser.escapedStringLiterals	false
+          > SELECT '%SystemDrive%\\USERS\\John' ilike r'%SystemDrive%\\Users%';
+          true
+          > SELECT '%SystemDrive%/Users/John' ilike '/%SYSTEMDrive/%//Users%' ESCAPE '/';
+          true
+      
+  note: |2-
+    
+        Use RLIKE to match with standard regular expressions.
+      
+  since: "3.3.0"
+
+- name: "in"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.In"
+  signatures:
+    - "expr1 in(expr2, expr3, ...)"
+  usage: "expr1 in(expr2, expr3, ...) - Returns true if `expr` equals to any valN."
+  arguments: |2-
+    
+        Arguments:
+          * expr1, expr2, expr3, ... - the arguments must be same type.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT 1 in(1, 2, 3);
+           true
+          > SELECT 1 in(2, 3, 4);
+           false
+          > SELECT named_struct('a', 1, 'b', 2) in(named_struct('a', 1, 'b', 1), named_struct('a', 1, 'b', 3));
+           false
+          > SELECT named_struct('a', 1, 'b', 2) in(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 3));
+           true
+      
+  since: "1.0.0"
+
+- name: "isnan"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.IsNaN"
+  signatures:
+    - "isnan(expr)"
+  usage: "isnan(expr) - Returns true if `expr` is NaN, or false otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT isnan(cast('NaN' as double));
+           true
+      
+  since: "1.5.0"
+
+- name: "isnotnull"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.IsNotNull"
+  signatures:
+    - "isnotnull(expr)"
+  usage: "isnotnull(expr) - Returns true if `expr` is not null, or false otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT isnotnull(1);
+           true
+      
+  since: "1.0.0"
+
+- name: "isnull"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.IsNull"
+  signatures:
+    - "isnull(expr)"
+  usage: "isnull(expr) - Returns true if `expr` is null, or false otherwise."
+  examples: |2-
+    
+        Examples:
+          > SELECT isnull(1);
+           false
+      
+  since: "1.0.0"
+
+- name: "like"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Like"
+  signatures:
+    - "str like pattern[ ESCAPE escape]"
+  usage: "str like pattern[ ESCAPE escape] - Returns true if str matches `pattern` with `escape`, null if any arguments are null, false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * pattern - a string expression. The pattern is a string which is matched literally, with
+              exception to the following special symbols:


+              _ matches any one character in the input (similar to . in posix regular expressions)\
+              % matches zero or more characters in the input (similar to .* in posix regular
+              expressions)


+              Since Spark 2.0, string literals are unescaped in our SQL parser, see the unescaping
+              rules at String Literal.
+              For example, in order to match "\abc", the pattern should be "\\abc".


+              When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back
+              to Spark 1.6 behavior regarding string literal parsing. For example, if the config is
+              enabled, the pattern to match "\abc" should be "\abc".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+          * escape - an character added since Spark 3.0. The default escape character is the '\'.
+              If an escape character precedes a special symbol or another escape character, the
+              following character is matched literally. It is invalid to escape any other character.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT like('Spark', '_park');
+          true
+          > SELECT '\\abc' AS S, S like r'\\abc', S like '\\\\abc';
+          \abc	true	true
+          > SET spark.sql.parser.escapedStringLiterals=true;
+          spark.sql.parser.escapedStringLiterals	true
+          > SELECT '%SystemDrive%\Users\John' like '\%SystemDrive\%\\Users%';
+          true
+          > SET spark.sql.parser.escapedStringLiterals=false;
+          spark.sql.parser.escapedStringLiterals	false
+          > SELECT '%SystemDrive%\\Users\\John' like r'%SystemDrive%\\Users%';
+          true
+          > SELECT '%SystemDrive%/Users/John' like '/%SystemDrive/%//Users%' ESCAPE '/';
+          true
+      
+  note: |2-
+    
+        Use RLIKE to match with standard regular expressions.
+      
+  since: "1.0.0"
+
+- name: "not"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Not"
+  signatures:
+    - "not expr"
+  usage: "not expr - Logical not."
+  examples: |2-
+    
+        Examples:
+          > SELECT not true;
+           false
+          > SELECT not false;
+           true
+          > SELECT not NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "or"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Or"
+  signatures:
+    - "expr1 or expr2"
+  usage: "expr1 or expr2 - Logical OR."
+  examples: |2-
+    
+        Examples:
+          > SELECT true or false;
+           true
+          > SELECT false or false;
+           false
+          > SELECT true or NULL;
+           true
+          > SELECT false or NULL;
+           NULL
+      
+  since: "1.0.0"
+
+- name: "regexp"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RLike"
+  signatures:
+    - "regexp(str, regexp)"
+  usage: "regexp(str, regexp) - Returns true if `str` matches `regexp`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * regexp - a string expression. The regex string should be a Java regular expression.
+    
+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".
+    
+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+      
+  examples: |2-
+    
+        Examples:
+          > SET spark.sql.parser.escapedStringLiterals=true;
+          spark.sql.parser.escapedStringLiterals	true
+          > SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');
+          true
+          > SET spark.sql.parser.escapedStringLiterals=false;
+          spark.sql.parser.escapedStringLiterals	false
+          > SELECT regexp('%SystemDrive%\\Users\\John', '%SystemDrive%\\\\Users.*');
+          true
+          > SELECT regexp('%SystemDrive%\\Users\\John', r'%SystemDrive%\\Users.*');
+          true
+      
+  note: |2-
+    
+        Use LIKE to match with simple string pattern.
+      
+  since: "3.2.0"
+
+- name: "regexp_like"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RLike"
+  signatures:
+    - "regexp_like(str, regexp)"
+  usage: "regexp_like(str, regexp) - Returns true if `str` matches `regexp`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * regexp - a string expression. The regex string should be a Java regular expression.
+    
+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".
+    
+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+      
+  examples: |2-
+    
+        Examples:
+          > SET spark.sql.parser.escapedStringLiterals=true;
+          spark.sql.parser.escapedStringLiterals	true
+          > SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');
+          true
+          > SET spark.sql.parser.escapedStringLiterals=false;
+          spark.sql.parser.escapedStringLiterals	false
+          > SELECT regexp_like('%SystemDrive%\\Users\\John', '%SystemDrive%\\\\Users.*');
+          true
+          > SELECT regexp_like('%SystemDrive%\\Users\\John', r'%SystemDrive%\\Users.*');
+          true
+      
+  note: |2-
+    
+        Use LIKE to match with simple string pattern.
+      
+  since: "3.2.0"
+
+- name: "rlike"
+  scope: "sql_builtin"
+  group: "predicate_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RLike"
+  signatures:
+    - "rlike(str, regexp)"
+  usage: "rlike(str, regexp) - Returns true if `str` matches `regexp`, or false otherwise."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * regexp - a string expression. The regex string should be a Java regular expression.
+    
+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".
+    
+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+      
+  examples: |2-
+    
+        Examples:
+          > SET spark.sql.parser.escapedStringLiterals=true;
+          spark.sql.parser.escapedStringLiterals	true
+          > SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');
+          true
+          > SET spark.sql.parser.escapedStringLiterals=false;
+          spark.sql.parser.escapedStringLiterals	false
+          > SELECT rlike('%SystemDrive%\\Users\\John', '%SystemDrive%\\\\Users.*');
+          true
+          > SELECT rlike('%SystemDrive%\\Users\\John', r'%SystemDrive%\\Users.*');
+          true
+      
+  note: |2-
+    
+        Use LIKE to match with simple string pattern.
+      
+  since: "1.0.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/string.yaml b/crates/sail-plan/data/functions/scalar/string.yaml
new file mode 100644
index 000000000..5b2c03ce3
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/string.yaml
@@ -0,0 +1,1902 @@
+- name: "ascii"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Ascii"
+  signatures:
+    - "ascii(str)"
+  usage: "ascii(str) - Returns the numeric value of the first character of `str`."
+  examples: |2-
+    
+        Examples:
+          > SELECT ascii('222');
+           50
+          > SELECT ascii(2);
+           50
+      
+  since: "1.5.0"
+
+- name: "base64"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Base64"
+  signatures:
+    - "base64(bin)"
+  usage: "base64(bin) - Converts the argument from a binary `bin` to a base 64 string."
+  examples: |2-
+    
+        Examples:
+          > SELECT base64('Spark SQL');
+           U3BhcmsgU1FM
+          > SELECT base64(x'537061726b2053514c');
+           U3BhcmsgU1FM
+      
+  since: "1.5.0"
+
+- name: "bit_length"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.BitLength"
+  signatures:
+    - "bit_length(expr)"
+  usage: "bit_length(expr) - Returns the bit length of string data or number of bits of binary data."
+  examples: |2-
+    
+        Examples:
+          > SELECT bit_length('Spark SQL');
+           72
+          > SELECT bit_length(x'537061726b2053514c');
+           72
+      
+  since: "2.3.0"
+
+- name: "btrim"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringTrimBoth"
+  signatures:
+    - "btrim(str)"
+    - "btrim(str, trimStr)"
+  usage: |2-
+    
+        btrim(str) - Removes the leading and trailing space characters from `str`.
+    
+        btrim(str, trimStr) - Remove the leading and trailing `trimStr` characters from `str`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * trimStr - the trim string characters to trim, the default value is a single space
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT btrim('    SparkSQL   ');
+           SparkSQL
+          > SELECT btrim(encode('    SparkSQL   ', 'utf-8'));
+           SparkSQL
+          > SELECT btrim('SSparkSQLS', 'SL');
+           parkSQ
+          > SELECT btrim(encode('SSparkSQLS', 'utf-8'), encode('SL', 'utf-8'));
+           parkSQ
+      
+  since: "3.2.0"
+
+- name: "char"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Chr"
+  signatures:
+    - "char(expr)"
+  usage: "char(expr) - Returns the ASCII character having the binary equivalent to `expr`. If n is larger than 256 the result is equivalent to chr(n % 256)"
+  examples: |2-
+    
+        Examples:
+          > SELECT char(65);
+           A
+      
+  since: "2.3.0"
+
+- name: "char_length"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Length"
+  signatures:
+    - "char_length(expr)"
+  usage: "char_length(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros."
+  examples: |2-
+    
+        Examples:
+          > SELECT char_length('Spark SQL ');
+           10
+          > SELECT char_length(x'537061726b2053514c');
+           9
+          > SELECT CHAR_LENGTH('Spark SQL ');
+           10
+          > SELECT CHARACTER_LENGTH('Spark SQL ');
+           10
+      
+  since: "2.3.0"
+
+- name: "character_length"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Length"
+  signatures:
+    - "character_length(expr)"
+  usage: "character_length(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros."
+  examples: |2-
+    
+        Examples:
+          > SELECT character_length('Spark SQL ');
+           10
+          > SELECT character_length(x'537061726b2053514c');
+           9
+          > SELECT CHAR_LENGTH('Spark SQL ');
+           10
+          > SELECT CHARACTER_LENGTH('Spark SQL ');
+           10
+      
+  since: "2.3.0"
+
+- name: "chr"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Chr"
+  signatures:
+    - "chr(expr)"
+  usage: "chr(expr) - Returns the ASCII character having the binary equivalent to `expr`. If n is larger than 256 the result is equivalent to chr(n % 256)"
+  examples: |2-
+    
+        Examples:
+          > SELECT chr(65);
+           A
+      
+  since: "2.3.0"
+
+- name: "collate"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CollateExpressionBuilder"
+  signatures:
+    - "collate(expr, collationName)"
+  usage: "collate(expr, collationName) - Marks a given expression with the specified collation."
+  arguments: |2-
+    
+        Arguments:
+          * expr - String expression to perform collation on.
+          * collationName - Foldable string expression that specifies the collation name.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT COLLATION('Spark SQL' collate UTF8_LCASE);
+          SYSTEM.BUILTIN.UTF8_LCASE
+      
+  since: "4.0.0"
+
+- name: "collation"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Collation"
+  signatures:
+    - "collation(expr)"
+  usage: "collation(expr) - Returns the collation name of a given expression."
+  arguments: |2-
+    
+        Arguments:
+          * expr - String expression to perform collation on.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT collation('Spark SQL');
+          SYSTEM.BUILTIN.UTF8_BINARY
+      
+  since: "4.0.0"
+
+- name: "concat_ws"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ConcatWs"
+  signatures:
+    - "concat_ws(sep[, str | array(str)]+)"
+  usage: "concat_ws(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`, skipping null values."
+  examples: |2-
+    
+        Examples:
+          > SELECT concat_ws(' ', 'Spark', 'SQL');
+            Spark SQL
+          > SELECT concat_ws('s');
+    
+          > SELECT concat_ws('/', 'foo', null, 'bar');
+            foo/bar
+          > SELECT concat_ws(null, 'Spark', 'SQL');
+            NULL
+      
+  since: "1.5.0"
+
+- name: "contains"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ContainsExpressionBuilder"
+  signatures:
+    - "contains(left, right)"
+  usage: |2-
+    
+        contains(left, right) - Returns a boolean. The value is True if right is found inside left.
+        Returns NULL if either input expression is NULL. Otherwise, returns False.
+        Both left or right must be of STRING or BINARY type.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT contains('Spark SQL', 'Spark');
+           true
+          > SELECT contains('Spark SQL', 'SPARK');
+           false
+          > SELECT contains('Spark SQL', null);
+           NULL
+          > SELECT contains(x'537061726b2053514c', x'537061726b');
+           true
+      
+  since: "3.3.0"
+
+- name: "decode"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Decode"
+  signatures:
+    - "decode(bin, charset)"
+    - "decode(expr, search, result [, search, result ] ... [, default])"
+  usage: |2-
+    
+        decode(bin, charset) - Decodes the first argument using the second argument character set. If either argument is null, the result will also be null.
+    
+        decode(expr, search, result [, search, result ] ... [, default]) - Compares expr
+          to each search value in order. If expr is equal to a search value, decode returns
+          the corresponding result. If no match is found, then it returns default. If default
+          is omitted, it returns null.
+      
+  arguments: |2-
+    
+        Arguments:
+          * bin - a binary expression to decode
+          * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to decode `bin` into a STRING. It is case insensitive.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT decode(encode('abc', 'utf-8'), 'utf-8');
+           abc
+          > SELECT decode(2, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic');
+           San Francisco
+          > SELECT decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle', 'Non domestic');
+           Non domestic
+          > SELECT decode(6, 1, 'Southlake', 2, 'San Francisco', 3, 'New Jersey', 4, 'Seattle');
+           NULL
+          > SELECT decode(null, 6, 'Spark', NULL, 'SQL', 4, 'rocks');
+           SQL
+      
+  note: |2-
+    
+        decode(expr, search, result [, search, result ] ... [, default]) is supported since 3.2.0
+      
+  since: "1.5.0"
+
+- name: "elt"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Elt"
+  signatures:
+    - "elt(n, input1, input2, ...)"
+  usage: |2-
+    
+        elt(n, input1, input2, ...) - Returns the `n`-th input, e.g., returns `input2` when `n` is 2.
+        The function returns NULL if the index exceeds the length of the array
+        and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true,
+        it throws ArrayIndexOutOfBoundsException for invalid indices.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT elt(1, 'scala', 'java');
+           scala
+          > SELECT elt(2, 'a', 1);
+           1
+      
+  since: "2.0.0"
+
+- name: "encode"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Encode"
+  signatures:
+    - "encode(str, charset)"
+  usage: "encode(str, charset) - Encodes the first argument using the second argument character set. If either argument is null, the result will also be null."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to encode `str` into a BINARY. It is case insensitive.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT encode('abc', 'utf-8');
+           abc
+      
+  since: "1.5.0"
+
+- name: "endswith"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.EndsWithExpressionBuilder"
+  signatures:
+    - "endswith(left, right)"
+  usage: |2-
+    
+        endswith(left, right) - Returns a boolean. The value is True if left ends with right.
+        Returns NULL if either input expression is NULL. Otherwise, returns False.
+        Both left or right must be of STRING or BINARY type.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT endswith('Spark SQL', 'SQL');
+           true
+          > SELECT endswith('Spark SQL', 'Spark');
+           false
+          > SELECT endswith('Spark SQL', null);
+           NULL
+          > SELECT endswith(x'537061726b2053514c', x'537061726b');
+           false
+          > SELECT endswith(x'537061726b2053514c', x'53514c');
+           true
+      
+  since: "3.3.0"
+
+- name: "find_in_set"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FindInSet"
+  signatures:
+    - "find_in_set(str, str_array)"
+  usage: |2-
+    
+        find_in_set(str, str_array) - Returns the index (1-based) of the given string (`str`) in the comma-delimited list (`str_array`).
+          Returns 0, if the string was not found or if the given string (`str`) contains a comma.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT find_in_set('ab','abc,b,ab,c,def');
+           3
+      
+  since: "1.5.0"
+
+- name: "format_number"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FormatNumber"
+  signatures:
+    - "format_number(expr1, expr2)"
+  usage: |2-
+    
+        format_number(expr1, expr2) - Formats the number `expr1` like '#,###,###.##', rounded to `expr2`
+          decimal places. If `expr2` is 0, the result has no decimal point or fractional part.
+          `expr2` also accept a user specified format.
+          This is supposed to function like MySQL's FORMAT.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT format_number(12332.123456, 4);
+           12,332.1235
+          > SELECT format_number(12332.123456, '##################.###');
+           12332.123
+      
+  since: "1.5.0"
+
+- name: "format_string"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FormatString"
+  signatures:
+    - "format_string(strfmt, obj, ...)"
+  usage: "format_string(strfmt, obj, ...) - Returns a formatted string from printf-style format strings."
+  examples: |2-
+    
+        Examples:
+          > SELECT format_string("Hello World %d %s", 100, "days");
+           Hello World 100 days
+      
+  since: "1.5.0"
+
+- name: "initcap"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.InitCap"
+  signatures:
+    - "initcap(str)"
+  usage: |2-
+    
+        initcap(str) - Returns `str` with the first letter of each word in uppercase.
+          All other letters are in lowercase. Words are delimited by white space.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT initcap('sPark sql');
+           Spark Sql
+      
+  since: "1.5.0"
+
+- name: "instr"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringInstr"
+  signatures:
+    - "instr(str, substr)"
+  usage: "instr(str, substr) - Returns the (1-based) index of the first occurrence of `substr` in `str`."
+  examples: |2-
+    
+        Examples:
+          > SELECT instr('SparkSQL', 'SQL');
+           6
+      
+  since: "1.5.0"
+
+- name: "is_valid_utf8"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.IsValidUTF8"
+  signatures:
+    - "is_valid_utf8(str)"
+  usage: "is_valid_utf8(str) - Returns true if `str` is a valid UTF-8 string, otherwise returns false."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT is_valid_utf8('Spark');
+           true
+          > SELECT is_valid_utf8(x'61');
+           true
+          > SELECT is_valid_utf8(x'80');
+           false
+          > SELECT is_valid_utf8(x'61C262');
+           false
+      
+  since: "4.0.0"
+
+- name: "lcase"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Lower"
+  signatures:
+    - "lcase(str)"
+  usage: "lcase(str) - Returns `str` with all characters changed to lowercase."
+  examples: |2-
+    
+        Examples:
+          > SELECT lcase('SparkSql');
+           sparksql
+      
+  since: "1.0.1"
+
+- name: "left"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Left"
+  signatures:
+    - "left(str, len)"
+  usage: "left(str, len) - Returns the leftmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string."
+  examples: |2-
+    
+        Examples:
+          > SELECT left('Spark SQL', 3);
+           Spa
+          > SELECT left(encode('Spark SQL', 'utf-8'), 3);
+           Spa
+      
+  since: "2.3.0"
+
+- name: "len"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Length"
+  signatures:
+    - "len(expr)"
+  usage: "len(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros."
+  examples: |2-
+    
+        Examples:
+          > SELECT len('Spark SQL ');
+           10
+          > SELECT len(x'537061726b2053514c');
+           9
+          > SELECT CHAR_LENGTH('Spark SQL ');
+           10
+          > SELECT CHARACTER_LENGTH('Spark SQL ');
+           10
+      
+  since: "3.4.0"
+
+- name: "length"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Length"
+  signatures:
+    - "length(expr)"
+  usage: "length(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros."
+  examples: |2-
+    
+        Examples:
+          > SELECT length('Spark SQL ');
+           10
+          > SELECT length(x'537061726b2053514c');
+           9
+          > SELECT CHAR_LENGTH('Spark SQL ');
+           10
+          > SELECT CHARACTER_LENGTH('Spark SQL ');
+           10
+      
+  since: "1.5.0"
+
+- name: "levenshtein"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Levenshtein"
+  signatures:
+    - "levenshtein(str1, str2[, threshold])"
+  usage: |2-
+    
+        levenshtein(str1, str2[, threshold]) - Returns the Levenshtein distance between the two given strings. If threshold is set and distance more than it, return -1.
+  examples: |2-
+    
+        Examples:
+          > SELECT levenshtein('kitten', 'sitting');
+           3
+          > SELECT levenshtein('kitten', 'sitting', 2);
+           -1
+      
+  since: "1.5.0"
+
+- name: "locate"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringLocate"
+  signatures:
+    - "locate(substr, str[, pos])"
+  usage: |2-
+    
+        locate(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`.
+          The given `pos` and return value are 1-based.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT locate('bar', 'foobarbar');
+           4
+          > SELECT locate('bar', 'foobarbar', 5);
+           7
+          > SELECT POSITION('bar' IN 'foobarbar');
+           4
+      
+  since: "1.5.0"
+
+- name: "lower"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Lower"
+  signatures:
+    - "lower(str)"
+  usage: "lower(str) - Returns `str` with all characters changed to lowercase."
+  examples: |2-
+    
+        Examples:
+          > SELECT lower('SparkSql');
+           sparksql
+      
+  since: "1.0.1"
+
+- name: "lpad"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.LPadExpressionBuilder"
+  signatures:
+    - "lpad(str, len[, pad])"
+  usage: |2-
+    
+        lpad(str, len[, pad]) - Returns `str`, left-padded with `pad` to a length of `len`.
+          If `str` is longer than `len`, the return value is shortened to `len` characters or bytes.
+          If `pad` is not specified, `str` will be padded to the left with space characters if it is
+          a character string, and with zeros if it is a byte sequence.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT lpad('hi', 5, '??');
+           ???hi
+          > SELECT lpad('hi', 1, '??');
+           h
+          > SELECT lpad('hi', 5);
+              hi
+          > SELECT hex(lpad(unhex('aabb'), 5));
+           000000AABB
+          > SELECT hex(lpad(unhex('aabb'), 5, unhex('1122')));
+           112211AABB
+      
+  since: "1.5.0"
+
+- name: "ltrim"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringTrimLeft"
+  signatures:
+    - "ltrim(str)"
+  usage: |2-
+    
+        ltrim(str) - Removes the leading space characters from `str`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * trimStr - the trim string characters to trim, the default value is a single space
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT ltrim('    SparkSQL   ');
+           SparkSQL
+      
+  since: "1.5.0"
+
+- name: "luhn_check"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Luhncheck"
+  signatures:
+    - "luhn_check(str )"
+  usage: |2-
+    
+        luhn_check(str ) - Checks that a string of digits is valid according to the Luhn algorithm.
+        This checksum function is widely applied on credit card numbers and government identification
+        numbers to distinguish valid numbers from mistyped, incorrect numbers.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT luhn_check('8112189876');
+           true
+          > SELECT luhn_check('79927398713');
+           true
+          > SELECT luhn_check('79927398714');
+           false
+      
+  since: "3.5.0"
+
+- name: "make_valid_utf8"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MakeValidUTF8"
+  signatures:
+    - "make_valid_utf8(str)"
+  usage: "make_valid_utf8(str) - Returns the original string if `str` is a valid UTF-8 string, otherwise returns a new string whose invalid UTF8 byte sequences are replaced using the UNICODE replacement character U+FFFD."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT make_valid_utf8('Spark');
+           Spark
+          > SELECT make_valid_utf8(x'61');
+           a
+          > SELECT make_valid_utf8(x'80');
+           �
+          > SELECT make_valid_utf8(x'61C262');
+           a�b
+      
+  since: "4.0.0"
+
+- name: "mask"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.MaskExpressionBuilder"
+  signatures:
+    - "mask(input[, upperChar, lowerChar, digitChar, otherChar])"
+  usage: |2-
+    mask(input[, upperChar, lowerChar, digitChar, otherChar]) - masks the given string value.
+           The function replaces characters with 'X' or 'x', and numbers with 'n'.
+           This can be useful for creating copies of tables with sensitive information removed.
+          
+  arguments: |2-
+    
+        Arguments:
+          * input      - string value to mask. Supported types: STRING, VARCHAR, CHAR
+          * upperChar  - character to replace upper-case characters with. Specify NULL to retain original character. Default value: 'X'
+          * lowerChar  - character to replace lower-case characters with. Specify NULL to retain original character. Default value: 'x'
+          * digitChar  - character to replace digit characters with. Specify NULL to retain original character. Default value: 'n'
+          * otherChar  - character to replace all other characters with. Specify NULL to retain original character. Default value: NULL
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT mask('abcd-EFGH-8765-4321');
+            xxxx-XXXX-nnnn-nnnn
+          > SELECT mask('abcd-EFGH-8765-4321', 'Q');
+            xxxx-QQQQ-nnnn-nnnn
+          > SELECT mask('AbCD123-@$#', 'Q', 'q');
+            QqQQnnn-@$#
+          > SELECT mask('AbCD123-@$#');
+            XxXXnnn-@$#
+          > SELECT mask('AbCD123-@$#', 'Q');
+            QxQQnnn-@$#
+          > SELECT mask('AbCD123-@$#', 'Q', 'q');
+            QqQQnnn-@$#
+          > SELECT mask('AbCD123-@$#', 'Q', 'q', 'd');
+            QqQQddd-@$#
+          > SELECT mask('AbCD123-@$#', 'Q', 'q', 'd', 'o');
+            QqQQdddoooo
+          > SELECT mask('AbCD123-@$#', NULL, 'q', 'd', 'o');
+            AqCDdddoooo
+          > SELECT mask('AbCD123-@$#', NULL, NULL, 'd', 'o');
+            AbCDdddoooo
+          > SELECT mask('AbCD123-@$#', NULL, NULL, NULL, 'o');
+            AbCD123oooo
+          > SELECT mask(NULL, NULL, NULL, NULL, 'o');
+            NULL
+          > SELECT mask(NULL);
+            NULL
+          > SELECT mask('AbCD123-@$#', NULL, NULL, NULL, NULL);
+            AbCD123-@$#
+      
+  since: "3.4.0"
+
+- name: "octet_length"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.OctetLength"
+  signatures:
+    - "octet_length(expr)"
+  usage: "octet_length(expr) - Returns the byte length of string data or number of bytes of binary data."
+  examples: |2-
+    
+        Examples:
+          > SELECT octet_length('Spark SQL');
+           9
+          > SELECT octet_length(x'537061726b2053514c');
+           9
+      
+  since: "2.3.0"
+
+- name: "overlay"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Overlay"
+  signatures:
+    - "overlay(input, replace, pos[, len])"
+  usage: "overlay(input, replace, pos[, len]) - Replace `input` with `replace` that starts at `pos` and is of length `len`."
+  examples: |2-
+    
+        Examples:
+          > SELECT overlay('Spark SQL' PLACING '_' FROM 6);
+           Spark_SQL
+          > SELECT overlay('Spark SQL' PLACING 'CORE' FROM 7);
+           Spark CORE
+          > SELECT overlay('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0);
+           Spark ANSI SQL
+          > SELECT overlay('Spark SQL' PLACING 'tructured' FROM 2 FOR 4);
+           Structured SQL
+          > SELECT overlay(encode('Spark SQL', 'utf-8') PLACING encode('_', 'utf-8') FROM 6);
+           Spark_SQL
+          > SELECT overlay(encode('Spark SQL', 'utf-8') PLACING encode('CORE', 'utf-8') FROM 7);
+           Spark CORE
+          > SELECT overlay(encode('Spark SQL', 'utf-8') PLACING encode('ANSI ', 'utf-8') FROM 7 FOR 0);
+           Spark ANSI SQL
+          > SELECT overlay(encode('Spark SQL', 'utf-8') PLACING encode('tructured', 'utf-8') FROM 2 FOR 4);
+           Structured SQL
+      
+  since: "3.0.0"
+
+- name: "position"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringLocate"
+  signatures:
+    - "position(substr, str[, pos])"
+  usage: |2-
+    
+        position(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`.
+          The given `pos` and return value are 1-based.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT position('bar', 'foobarbar');
+           4
+          > SELECT position('bar', 'foobarbar', 5);
+           7
+          > SELECT POSITION('bar' IN 'foobarbar');
+           4
+      
+  since: "2.3.0"
+
+- name: "printf"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.FormatString"
+  signatures:
+    - "printf(strfmt, obj, ...)"
+  usage: "printf(strfmt, obj, ...) - Returns a formatted string from printf-style format strings."
+  examples: |2-
+    
+        Examples:
+          > SELECT printf("Hello World %d %s", 100, "days");
+           Hello World 100 days
+      
+  since: "1.5.0"
+
+- name: "quote"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Quote"
+  signatures:
+    - "quote(str)"
+  usage: "quote(str) - Returns `str` enclosed by single quotes and each instance of single quote in it is preceded by a backslash."
+  examples: |2-
+    
+        Examples:
+          > SELECT quote('Don\'t');
+           'Don\'t'
+      
+  since: "4.1.0"
+
+- name: "randstr"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RandStr"
+  signatures:
+    - "randstr(length[, seed])"
+  usage: |2-
+    
+        randstr(length[, seed]) - Returns a string of the specified length whose characters are chosen
+          uniformly at random from the following pool of characters: 0-9, a-z, A-Z. The random seed is
+          optional. The string length must be a constant two-byte or four-byte integer (SMALLINT or INT,
+          respectively).
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT randstr(3, 0) AS result;
+           ceV
+      
+  since: "4.0.0"
+
+- name: "regexp_count"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RegExpCount"
+  signatures:
+    - "regexp_count(str, regexp)"
+  usage: |2-
+    
+        regexp_count(str, regexp) - Returns a count of the number of times that the regular expression pattern `regexp` is matched in the string `str`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression.
+          * regexp - a string representing a regular expression. The regex string should be a
+              Java regular expression.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en');
+           2
+          > SELECT regexp_count('abcdefghijklmnopqrstuvwxyz', '[a-z]{3}');
+           8
+      
+  since: "3.4.0"
+
+- name: "regexp_extract"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RegExpExtract"
+  signatures:
+    - "regexp_extract(str, regexp[, idx])"
+  usage: |2-
+    
+        regexp_extract(str, regexp[, idx]) - Extract the first string in the `str` that match the `regexp`
+        expression and corresponding to the regex group index.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression.
+          * regexp - a string representing a regular expression. The regex string should be a
+              Java regular expression.


+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".


+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+          * idx - an integer expression that representing the group index. The regex maybe contains
+              multiple groups. `idx` indicates which regex group to extract. The group index should
+              be non-negative. The minimum value of `idx` is 0, which means matching the entire
+              regular expression. If `idx` is not specified, the default group index value is 1. The
+              `idx` parameter is the Java regex Matcher group() method index.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1);
+           100
+          > SELECT regexp_extract('100-200', r'(\d+)-(\d+)', 1);
+           100
+      
+  since: "1.5.0"
+
+- name: "regexp_extract_all"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RegExpExtractAll"
+  signatures:
+    - "regexp_extract_all(str, regexp[, idx])"
+  usage: |2-
+    
+        regexp_extract_all(str, regexp[, idx]) - Extract all strings in the `str` that match the `regexp`
+        expression and corresponding to the regex group index.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression.
+          * regexp - a string representing a regular expression. The regex string should be a
+              Java regular expression.


+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".


+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+          * idx - an integer expression that representing the group index. The regex may contains
+              multiple groups. `idx` indicates which regex group to extract. The group index should
+              be non-negative. The minimum value of `idx` is 0, which means matching the entire
+              regular expression. If `idx` is not specified, the default group index value is 1. The
+              `idx` parameter is the Java regex Matcher group() method index.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1);
+           ["100","300"]
+          > SELECT regexp_extract_all('100-200, 300-400', r'(\d+)-(\d+)', 1);
+           ["100","300"]
+      
+  since: "3.1.0"
+
+- name: "regexp_instr"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RegExpInStr"
+  signatures:
+    - "regexp_instr(str, regexp)"
+  usage: |2-
+    
+        regexp_instr(str, regexp) - Searches a string for a regular expression and returns an integer that indicates the beginning position of the matched substring. Positions are 1-based, not 0-based. If no match is found, returns 0.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression.
+          * regexp - a string representing a regular expression. The regex string should be a
+              Java regular expression.


+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".


+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT regexp_instr(r"\abc", r"^\\abc$");
+           1
+          > SELECT regexp_instr('user@spark.apache.org', '@[^.]*');
+           5
+      
+  since: "3.4.0"
+
+- name: "regexp_replace"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RegExpReplace"
+  signatures:
+    - "regexp_replace(str, regexp, rep[, position])"
+  usage: "regexp_replace(str, regexp, rep[, position]) - Replaces all substrings of `str` that match `regexp` with `rep`."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression to search for a regular expression pattern match.
+          * regexp - a string representing a regular expression. The regex string should be a
+              Java regular expression.


+              Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL
+              parser, see the unescaping rules at String Literal.
+              For example, to match "\abc", a regular expression for `regexp` can be "^\\abc$".


+              There is a SQL config 'spark.sql.parser.escapedStringLiterals' that can be used to
+              fallback to the Spark 1.6 behavior regarding string literal parsing. For example,
+              if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".


+              It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping
+              special characters in the pattern string if exists.
+          * rep - a string expression to replace matched substrings.
+          * position - a positive integer literal that indicates the position within `str` to begin searching.
+              The default is 1. If position is greater than the number of characters in `str`, the result is `str`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT regexp_replace('100-200', '(\\d+)', 'num');
+           num-num
+          > SELECT regexp_replace('100-200', r'(\d+)', 'num');
+           num-num
+      
+  since: "1.5.0"
+
+- name: "regexp_substr"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RegExpSubStr"
+  signatures:
+    - "regexp_substr(str, regexp)"
+  usage: |2-
+    
+        regexp_substr(str, regexp) - Returns the substring that matches the regular expression `regexp` within the string `str`. If the regular expression is not found, the result is null.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression.
+          * regexp - a string representing a regular expression. The regex string should be a Java regular expression.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en');
+           Steven
+          > SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Jeck');
+           NULL
+      
+  since: "3.4.0"
+
+- name: "repeat"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringRepeat"
+  signatures:
+    - "repeat(str, n)"
+  usage: "repeat(str, n) - Returns the string which repeats the given string value n times."
+  examples: |2-
+    
+        Examples:
+          > SELECT repeat('123', 2);
+           123123
+      
+  since: "1.5.0"
+
+- name: "replace"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringReplace"
+  signatures:
+    - "replace(str, search[, replace])"
+  usage: "replace(str, search[, replace]) - Replaces all occurrences of `search` with `replace`."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * search - a string expression. If `search` is not found in `str`, `str` is returned unchanged.
+          * replace - a string expression. If `replace` is not specified or is an empty string, nothing replaces
+              the string that is removed from `str`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT replace('ABCabc', 'abc', 'DEF');
+           ABCDEF
+      
+  since: "2.3.0"
+
+- name: "right"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Right"
+  signatures:
+    - "right(str, len)"
+  usage: "right(str, len) - Returns the rightmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string."
+  examples: |2-
+    
+        Examples:
+          > SELECT right('Spark SQL', 3);
+           SQL
+      
+  since: "2.3.0"
+
+- name: "rpad"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RPadExpressionBuilder"
+  signatures:
+    - "rpad(str, len[, pad])"
+  usage: |2-
+    
+        rpad(str, len[, pad]) - Returns `str`, right-padded with `pad` to a length of `len`.
+          If `str` is longer than `len`, the return value is shortened to `len` characters.
+          If `pad` is not specified, `str` will be padded to the right with space characters if it is
+          a character string, and with zeros if it is a binary string.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT rpad('hi', 5, '??');
+           hi???
+          > SELECT rpad('hi', 1, '??');
+           h
+          > SELECT rpad('hi', 5);
+           hi
+          > SELECT hex(rpad(unhex('aabb'), 5));
+           AABB000000
+          > SELECT hex(rpad(unhex('aabb'), 5, unhex('1122')));
+           AABB112211
+      
+  since: "1.5.0"
+
+- name: "rtrim"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringTrimRight"
+  signatures:
+    - "rtrim(str)"
+  usage: |2-
+    
+        rtrim(str) - Removes the trailing space characters from `str`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * trimStr - the trim string characters to trim, the default value is a single space
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT rtrim('    SparkSQL   ');
+           SparkSQL
+      
+  since: "1.5.0"
+
+- name: "sentences"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Sentences"
+  signatures:
+    - "sentences(str[, lang[, country]])"
+  usage: "sentences(str[, lang[, country]]) - Splits `str` into an array of array of words."
+  arguments: |2-
+    
+        Arguments:
+          * str - A STRING expression to be parsed.
+          * lang - An optional STRING expression with a language code from ISO 639 Alpha-2 (e.g. 'DE'),
+              Alpha-3, or a language subtag of up to 8 characters.
+          * country - An optional STRING expression with a country code from ISO 3166 alpha-2 country
+              code or a UN M.49 numeric-3 area code.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT sentences('Hi there! Good morning.');
+           [["Hi","there"],["Good","morning"]]
+          > SELECT sentences('Hi there! Good morning.', 'en');
+           [["Hi","there"],["Good","morning"]]
+          > SELECT sentences('Hi there! Good morning.', 'en', 'US');
+           [["Hi","there"],["Good","morning"]]
+      
+  since: "2.0.0"
+
+- name: "soundex"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SoundEx"
+  signatures:
+    - "soundex(str)"
+  usage: "soundex(str) - Returns Soundex code of the string."
+  examples: |2-
+    
+        Examples:
+          > SELECT soundex('Miller');
+           M460
+      
+  since: "1.5.0"
+
+- name: "space"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringSpace"
+  signatures:
+    - "space(n)"
+  usage: "space(n) - Returns a string consisting of `n` spaces."
+  examples: |2-
+    
+        Examples:
+          > SELECT concat(space(2), '1');
+             1
+      
+  since: "1.5.0"
+
+- name: "split"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringSplit"
+  signatures:
+    - "split(str, regex, limit)"
+  usage: "split(str, regex, limit) - Splits `str` around occurrences that match `regex` and returns an array with a length of at most `limit`"
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression to split.
+          * regex - a string representing a regular expression. The regex string should be a
+            Java regular expression.
+          * limit - an integer expression which controls the number of times the regex is applied.
+              * limit > 0: The resulting array's length will not be more than `limit`,
+                and the resulting array's last entry will contain all input
+                beyond the last matched regex.
+              * limit <= 0: `regex` will be applied as many times as possible, and
+                the resulting array can be of any size.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT split('oneAtwoBthreeC', '[ABC]');
+           ["one","two","three",""]
+          > SELECT split('oneAtwoBthreeC', '[ABC]', -1);
+           ["one","two","three",""]
+          > SELECT split('oneAtwoBthreeC', '[ABC]', 2);
+           ["one","twoBthreeC"]
+      
+  since: "1.5.0"
+
+- name: "split_part"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SplitPart"
+  signatures:
+    - "split_part(str, delimiter, partNum)"
+  usage: |2-
+    
+        split_part(str, delimiter, partNum) - Splits `str` by delimiter and return
+          requested part of the split (1-based). If any input is null, returns null.
+          if `partNum` is out of range of split parts, returns empty string. If `partNum` is 0,
+          throws an error. If `partNum` is negative, the parts are counted backward from the
+          end of the string. If the `delimiter` is an empty string, the `str` is not split.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT split_part('11.12.13', '.', 3);
+           13
+      
+  since: "3.3.0"
+
+- name: "startswith"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StartsWithExpressionBuilder"
+  signatures:
+    - "startswith(left, right)"
+  usage: |2-
+    
+        startswith(left, right) - Returns a boolean. The value is True if left starts with right.
+        Returns NULL if either input expression is NULL. Otherwise, returns False.
+        Both left or right must be of STRING or BINARY type.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT startswith('Spark SQL', 'Spark');
+           true
+          > SELECT startswith('Spark SQL', 'SQL');
+           false
+          > SELECT startswith('Spark SQL', null);
+           NULL
+          > SELECT startswith(x'537061726b2053514c', x'537061726b');
+           true
+          > SELECT startswith(x'537061726b2053514c', x'53514c');
+           false
+      
+  since: "3.3.0"
+
+- name: "strpos"
+  scope: "sail_extension"
+  signatures:
+    - "strpos(str, substr)"
+  usage: "strpos(str, substr) - Returns the 1-based position of substr in str, or 0 if substr is not found."
+
+- name: "substr"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Substring"
+  signatures:
+    - "substr(str, pos[, len])"
+    - "substr(str FROM pos[ FOR len]])"
+  usage: |2-
+    
+        substr(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+    
+        substr(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT substr('Spark SQL', 5);
+           k SQL
+          > SELECT substr('Spark SQL', -3);
+           SQL
+          > SELECT substr('Spark SQL', 5, 1);
+           k
+          > SELECT substr('Spark SQL' FROM 5);
+           k SQL
+          > SELECT substr('Spark SQL' FROM -3);
+           SQL
+          > SELECT substr('Spark SQL' FROM 5 FOR 1);
+           k
+          > SELECT substr(encode('Spark SQL', 'utf-8'), 5);
+           k SQL
+      
+  since: "1.5.0"
+
+- name: "substring"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Substring"
+  signatures:
+    - "substring(str, pos[, len])"
+    - "substring(str FROM pos[ FOR len]])"
+  usage: |2-
+    
+        substring(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+    
+        substring(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT substring('Spark SQL', 5);
+           k SQL
+          > SELECT substring('Spark SQL', -3);
+           SQL
+          > SELECT substring('Spark SQL', 5, 1);
+           k
+          > SELECT substring('Spark SQL' FROM 5);
+           k SQL
+          > SELECT substring('Spark SQL' FROM -3);
+           SQL
+          > SELECT substring('Spark SQL' FROM 5 FOR 1);
+           k
+          > SELECT substring(encode('Spark SQL', 'utf-8'), 5);
+           k SQL
+      
+  since: "1.5.0"
+
+- name: "substring_index"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SubstringIndex"
+  signatures:
+    - "substring_index(str, delim, count)"
+  usage: |2-
+    
+        substring_index(str, delim, count) - Returns the substring from `str` before `count` occurrences of the delimiter `delim`.
+          If `count` is positive, everything to the left of the final delimiter (counting from the
+          left) is returned. If `count` is negative, everything to the right of the final delimiter
+          (counting from the right) is returned. The function substring_index performs a case-sensitive match
+          when searching for `delim`.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT substring_index('www.apache.org', '.', 2);
+           www.apache
+      
+  since: "1.5.0"
+
+- name: "to_binary"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToBinary"
+  signatures:
+    - "to_binary(str[, fmt])"
+  usage: |2-
+    
+        to_binary(str[, fmt]) - Converts the input `str` to a binary value based on the supplied `fmt`.
+          `fmt` can be a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64".
+          By default, the binary format for conversion is "hex" if `fmt` is omitted.
+          The function returns NULL if at least one of the input parameters is NULL.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_binary('abc', 'utf-8');
+           abc
+      
+  since: "3.3.0"
+
+- name: "to_char"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToCharacterBuilder"
+  signatures:
+    - "to_char(expr, format)"
+  usage: |2-
+    
+        to_char(expr, format) - Convert `expr` to a string based on the `format`.
+          Throws an exception if the conversion fails. The format can consist of the following
+          characters, case insensitive:
+            '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
+              string matches a sequence of digits in the input value, generating a result string of the
+              same length as the corresponding sequence in the format string. The result string is
+              left-padded with zeros if the 0/9 sequence comprises more digits than the matching part of
+              the decimal value, starts with 0, and is before the decimal point. Otherwise, it is
+              padded with spaces.
+            '.' or 'D': Specifies the position of the decimal point (optional, only allowed once).
+            ',' or 'G': Specifies the position of the grouping (thousands) separator (,). There must be
+              a 0 or 9 to the left and right of each grouping separator.
+            '$': Specifies the location of the $ currency sign. This character may only be specified
+              once.
+            'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at
+              the beginning or end of the format string). Note that 'S' prints '+' for positive values
+              but 'MI' prints a space.
+            'PR': Only allowed at the end of the format string; specifies that the result string will be
+              wrapped by angle brackets if the input value is negative.
+              ('<1>').
+          If `expr` is a datetime, `format` shall be a valid datetime pattern, see Datetime Patterns.
+          If `expr` is a binary, it is converted to a string in one of the formats:
+            'base64': a base 64 string.
+            'hex': a string in the hexadecimal format.
+            'utf-8': the input binary is decoded to UTF-8 string.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_char(454, '999');
+           454
+          > SELECT to_char(454.00, '000D00');
+           454.00
+          > SELECT to_char(12454, '99G999');
+           12,454
+          > SELECT to_char(78.12, '$99.99');
+           $78.12
+          > SELECT to_char(-12454.8, '99G999D9S');
+           12,454.8-
+          > SELECT to_char(date'2016-04-08', 'y');
+           2016
+          > SELECT to_char(x'537061726b2053514c', 'base64');
+           U3BhcmsgU1FM
+          > SELECT to_char(x'537061726b2053514c', 'hex');
+           537061726B2053514C
+          > SELECT to_char(encode('abc', 'utf-8'), 'utf-8');
+           abc
+      
+  since: "3.4.0"
+
+- name: "to_number"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToNumber"
+  signatures:
+    - "to_number(expr, fmt)"
+  usage: |2-
+    
+         to_number(expr, fmt) - Convert string 'expr' to a number based on the string format 'fmt'.
+           Throws an exception if the conversion fails. The format can consist of the following
+           characters, case insensitive:
+             '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
+               string matches a sequence of digits in the input string. If the 0/9 sequence starts with
+               0 and is before the decimal point, it can only match a digit sequence of the same size.
+               Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a
+               digit sequence that has the same or smaller size.
+             '.' or 'D': Specifies the position of the decimal point (optional, only allowed once).
+             ',' or 'G': Specifies the position of the grouping (thousands) separator (,). There must be
+               a 0 or 9 to the left and right of each grouping separator. 'expr' must match the
+               grouping separator relevant for the size of the number.
+             '$': Specifies the location of the $ currency sign. This character may only be specified
+               once.
+             'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at
+               the beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not.
+             'PR': Only allowed at the end of the format string; specifies that 'expr' indicates a
+               negative number with wrapping angled brackets.
+               ('<1>').
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_number('454', '999');
+           454
+          > SELECT to_number('454.00', '000.00');
+           454.00
+          > SELECT to_number('12,454', '99,999');
+           12454
+          > SELECT to_number('$78.12', '$99.99');
+           78.12
+          > SELECT to_number('12,454.8-', '99,999.9S');
+           -12454.8
+      
+  since: "3.3.0"
+
+- name: "to_varchar"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ToCharacterBuilder"
+  signatures:
+    - "to_varchar(expr, format)"
+  usage: |2-
+    
+        to_varchar(expr, format) - Convert `expr` to a string based on the `format`.
+          Throws an exception if the conversion fails. The format can consist of the following
+          characters, case insensitive:
+            '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
+              string matches a sequence of digits in the input value, generating a result string of the
+              same length as the corresponding sequence in the format string. The result string is
+              left-padded with zeros if the 0/9 sequence comprises more digits than the matching part of
+              the decimal value, starts with 0, and is before the decimal point. Otherwise, it is
+              padded with spaces.
+            '.' or 'D': Specifies the position of the decimal point (optional, only allowed once).
+            ',' or 'G': Specifies the position of the grouping (thousands) separator (,). There must be
+              a 0 or 9 to the left and right of each grouping separator.
+            '$': Specifies the location of the $ currency sign. This character may only be specified
+              once.
+            'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at
+              the beginning or end of the format string). Note that 'S' prints '+' for positive values
+              but 'MI' prints a space.
+            'PR': Only allowed at the end of the format string; specifies that the result string will be
+              wrapped by angle brackets if the input value is negative.
+              ('<1>').
+          If `expr` is a datetime, `format` shall be a valid datetime pattern, see Datetime Patterns.
+          If `expr` is a binary, it is converted to a string in one of the formats:
+            'base64': a base 64 string.
+            'hex': a string in the hexadecimal format.
+            'utf-8': the input binary is decoded to UTF-8 string.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT to_varchar(454, '999');
+           454
+          > SELECT to_varchar(454.00, '000D00');
+           454.00
+          > SELECT to_varchar(12454, '99G999');
+           12,454
+          > SELECT to_varchar(78.12, '$99.99');
+           $78.12
+          > SELECT to_varchar(-12454.8, '99G999D9S');
+           12,454.8-
+          > SELECT to_varchar(date'2016-04-08', 'y');
+           2016
+          > SELECT to_varchar(x'537061726b2053514c', 'base64');
+           U3BhcmsgU1FM
+          > SELECT to_varchar(x'537061726b2053514c', 'hex');
+           537061726B2053514C
+          > SELECT to_varchar(encode('abc', 'utf-8'), 'utf-8');
+           abc
+      
+  since: "3.5.0"
+
+- name: "translate"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringTranslate"
+  signatures:
+    - "translate(input, from, to)"
+  usage: "translate(input, from, to) - Translates the `input` string by replacing the characters present in the `from` string with the corresponding characters in the `to` string."
+  examples: |2-
+    
+        Examples:
+          > SELECT translate('AaBbCc', 'abc', '123');
+           A1B2C3
+      
+  since: "1.5.0"
+
+- name: "trim"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StringTrim"
+  signatures:
+    - "trim(str)"
+    - "trim(BOTH FROM str)"
+    - "trim(LEADING FROM str)"
+    - "trim(TRAILING FROM str)"
+    - "trim(trimStr FROM str)"
+    - "trim(BOTH trimStr FROM str)"
+    - "trim(LEADING trimStr FROM str)"
+    - "trim(TRAILING trimStr FROM str)"
+  usage: |2-
+    
+        trim(str) - Removes the leading and trailing space characters from `str`.
+    
+        trim(BOTH FROM str) - Removes the leading and trailing space characters from `str`.
+    
+        trim(LEADING FROM str) - Removes the leading space characters from `str`.
+    
+        trim(TRAILING FROM str) - Removes the trailing space characters from `str`.
+    
+        trim(trimStr FROM str) - Remove the leading and trailing `trimStr` characters from `str`.
+    
+        trim(BOTH trimStr FROM str) - Remove the leading and trailing `trimStr` characters from `str`.
+    
+        trim(LEADING trimStr FROM str) - Remove the leading `trimStr` characters from `str`.
+    
+        trim(TRAILING trimStr FROM str) - Remove the trailing `trimStr` characters from `str`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+          * trimStr - the trim string characters to trim, the default value is a single space
+          * BOTH, FROM - these are keywords to specify trimming string characters from both ends of
+              the string
+          * LEADING, FROM - these are keywords to specify trimming string characters from the left
+              end of the string
+          * TRAILING, FROM - these are keywords to specify trimming string characters from the right
+              end of the string
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT trim('    SparkSQL   ');
+           SparkSQL
+          > SELECT trim(BOTH FROM '    SparkSQL   ');
+           SparkSQL
+          > SELECT trim(LEADING FROM '    SparkSQL   ');
+           SparkSQL
+          > SELECT trim(TRAILING FROM '    SparkSQL   ');
+               SparkSQL
+          > SELECT trim('SL' FROM 'SSparkSQLS');
+           parkSQ
+          > SELECT trim(BOTH 'SL' FROM 'SSparkSQLS');
+           parkSQ
+          > SELECT trim(LEADING 'SL' FROM 'SSparkSQLS');
+           parkSQLS
+          > SELECT trim(TRAILING 'SL' FROM 'SSparkSQLS');
+           SSparkSQ
+      
+  since: "1.5.0"
+
+- name: "try_to_binary"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryToBinary"
+  signatures:
+    - "try_to_binary(str[, fmt])"
+  usage: "try_to_binary(str[, fmt]) - This is a special version of `to_binary` that performs the same operation, but returns a NULL value instead of raising an error if the conversion cannot be performed."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_to_binary('abc', 'utf-8');
+           abc
+          > select try_to_binary('a!', 'base64');
+           NULL
+          > select try_to_binary('abc', 'invalidFormat');
+           NULL
+      
+  since: "3.3.0"
+
+- name: "try_to_number"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryToNumber"
+  signatures:
+    - "try_to_number(expr, fmt)"
+  usage: |2-
+    
+         try_to_number(expr, fmt) - Convert string 'expr' to a number based on the string format `fmt`.
+           Returns NULL if the string 'expr' does not match the expected format. The format follows the
+           same semantics as the to_number function.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_to_number('454', '999');
+           454
+          > SELECT try_to_number('454.00', '000.00');
+           454.00
+          > SELECT try_to_number('12,454', '99,999');
+           12454
+          > SELECT try_to_number('$78.12', '$99.99');
+           78.12
+          > SELECT try_to_number('12,454.8-', '99,999.9S');
+           -12454.8
+      
+  since: "3.3.0"
+
+- name: "try_validate_utf8"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryValidateUTF8"
+  signatures:
+    - "try_validate_utf8(str)"
+  usage: "try_validate_utf8(str) - Returns the original string if `str` is a valid UTF-8 string, otherwise returns NULL."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_validate_utf8('Spark');
+           Spark
+          > SELECT try_validate_utf8(x'61');
+           a
+          > SELECT try_validate_utf8(x'80');
+           NULL
+          > SELECT try_validate_utf8(x'61C262');
+           NULL
+      
+  since: "4.0.0"
+
+- name: "ucase"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Upper"
+  signatures:
+    - "ucase(str)"
+  usage: "ucase(str) - Returns `str` with all characters changed to uppercase."
+  examples: |2-
+    
+        Examples:
+          > SELECT ucase('SparkSql');
+           SPARKSQL
+      
+  since: "1.0.1"
+
+- name: "unbase64"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UnBase64"
+  signatures:
+    - "unbase64(str)"
+  usage: "unbase64(str) - Converts the argument from a base 64 string `str` to a binary."
+  examples: |2-
+    
+        Examples:
+          > SELECT unbase64('U3BhcmsgU1FM');
+           Spark SQL
+      
+  since: "1.5.0"
+
+- name: "upper"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Upper"
+  signatures:
+    - "upper(str)"
+  usage: "upper(str) - Returns `str` with all characters changed to uppercase."
+  examples: |2-
+    
+        Examples:
+          > SELECT upper('SparkSql');
+           SPARKSQL
+      
+  since: "1.0.1"
+
+- name: "validate_utf8"
+  scope: "sql_builtin"
+  group: "string_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ValidateUTF8"
+  signatures:
+    - "validate_utf8(str)"
+  usage: "validate_utf8(str) - Returns the original string if `str` is a valid UTF-8 string, otherwise throws an exception."
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT validate_utf8('Spark');
+           Spark
+          > SELECT validate_utf8(x'61');
+           a
+      
+  since: "4.0.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/struct.yaml b/crates/sail-plan/data/functions/scalar/struct.yaml
new file mode 100644
index 000000000..942d3b3a9
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/struct.yaml
@@ -0,0 +1,32 @@
+- name: "named_struct"
+  scope: "sql_builtin"
+  group: "struct_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CreateNamedStruct"
+  signatures:
+    - "named_struct(name1, val1, name2, val2, ...)"
+  usage: "named_struct(name1, val1, name2, val2, ...) - Creates a struct with the given field names and values."
+  examples: |2-
+    
+        Examples:
+          > SELECT named_struct("a", 1, "b", 2, "c", 3);
+           {"a":1,"b":2,"c":3}
+      
+  since: "1.5.0"
+
+- name: "struct"
+  scope: "sql_builtin"
+  group: "struct_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CreateNamedStruct"
+  signatures:
+    - "struct(col1, col2, col3, ...)"
+  usage: "struct(col1, col2, col3, ...) - Creates a struct with the given field values."
+  examples: |2-
+    
+        Examples:
+          > SELECT struct(1, 2, 3);
+           {"col1":1,"col2":2,"col3":3}
+      
+  since: "1.4.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/url.yaml b/crates/sail-plan/data/functions/scalar/url.yaml
new file mode 100644
index 000000000..60d6d3bd1
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/url.yaml
@@ -0,0 +1,114 @@
+- name: "parse_url"
+  scope: "sql_builtin"
+  group: "url_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.ParseUrl"
+  signatures:
+    - "parse_url(url, partToExtract[, key])"
+  usage: "parse_url(url, partToExtract[, key]) - Extracts a part from a URL."
+  examples: |2-
+    
+        Examples:
+          > SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');
+           spark.apache.org
+          > SELECT parse_url('http://spark.apache.org/path?query=1', 'QUERY');
+           query=1
+          > SELECT parse_url('http://spark.apache.org/path?query=1', 'QUERY', 'query');
+           1
+      
+  since: "2.0.0"
+
+- name: "try_parse_url"
+  scope: "sql_builtin"
+  group: "url_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryParseUrl"
+  signatures:
+    - "try_parse_url(url, partToExtract[, key])"
+  usage: "try_parse_url(url, partToExtract[, key]) - This is a special version of `parse_url` that performs the same operation, but returns a NULL value instead of raising an error if the parsing cannot be performed."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_parse_url('http://spark.apache.org/path?query=1', 'HOST');
+           spark.apache.org
+          > SELECT try_parse_url('http://spark.apache.org/path?query=1', 'QUERY');
+           query=1
+          > SELECT try_parse_url('inva lid://spark.apache.org/path?query=1', 'QUERY');
+           NULL
+          > SELECT try_parse_url('http://spark.apache.org/path?query=1', 'QUERY', 'query');
+           1
+      
+  since: "4.0.0"
+
+- name: "try_url_decode"
+  scope: "sql_builtin"
+  group: "url_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.TryUrlDecode"
+  signatures:
+    - "try_url_decode(str)"
+  usage: |2-
+    
+        try_url_decode(str) - This is a special version of `url_decode` that performs the same operation, but returns a NULL value instead of raising an error if the decoding cannot be performed.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression to decode
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT try_url_decode('https%3A%2F%2Fspark.apache.org');
+           https://spark.apache.org
+      
+  since: "4.0.0"
+
+- name: "url_decode"
+  scope: "sql_builtin"
+  group: "url_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UrlDecode"
+  signatures:
+    - "url_decode(str)"
+  usage: |2-
+    
+        url_decode(str) - Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding scheme.
+      
+  arguments: |2-
+    
+        Arguments:
+          * str - a string expression to decode
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT url_decode('https%3A%2F%2Fspark.apache.org');
+           https://spark.apache.org
+      
+  since: "3.4.0"
+
+- name: "url_encode"
+  scope: "sql_builtin"
+  group: "url_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.UrlEncode"
+  signatures:
+    - "url_encode(str)"
+  usage: |2-
+    
+        url_encode(str) - Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding scheme.
+      
+  arguments: |2-
+    
+        Arguments:
+          str - a string expression to be translated
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT url_encode('https://spark.apache.org');
+           https%3A%2F%2Fspark.apache.org
+      
+  since: "3.4.0"
+
diff --git a/crates/sail-plan/data/functions/scalar/variant.yaml b/crates/sail-plan/data/functions/scalar/variant.yaml
new file mode 100644
index 000000000..e133d0d97
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/variant.yaml
@@ -0,0 +1,191 @@
+- name: "is_valid_variant"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.IsValidVariant"
+  signatures:
+    - "is_valid_variant(v)"
+  usage: "is_valid_variant(v) - Returns true if the variant is valid, false if it is malformed, NULL if `v` is NULL."
+  examples: |2-
+    
+        Examples:
+          > SELECT is_valid_variant(parse_json('null'));
+           true
+          > SELECT is_valid_variant(parse_json('[{"b":true,"a":0}]'));
+           true
+      
+  since: "4.2.0"
+
+- name: "is_variant_null"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.IsVariantNull"
+  signatures:
+    - "is_variant_null(expr)"
+  usage: "is_variant_null(expr) - Check if a variant value is a variant null. Returns true if and only if the input is a variant null and false otherwise (including in the case of SQL NULL)."
+  examples: |2-
+    
+        Examples:
+          > SELECT is_variant_null(parse_json('null'));
+           true
+          > SELECT is_variant_null(parse_json('"null"'));
+           false
+          > SELECT is_variant_null(parse_json('13'));
+           false
+          > SELECT is_variant_null(parse_json(null));
+           false
+          > SELECT is_variant_null(variant_get(parse_json('{"a":null, "b":"spark"}'), "$.c"));
+           false
+          > SELECT is_variant_null(variant_get(parse_json('{"a":null, "b":"spark"}'), "$.a"));
+           true
+      
+  since: "4.0.0"
+
+- name: "parse_json"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.ParseJsonExpressionBuilder"
+  signatures:
+    - "parse_json(jsonStr)"
+  usage: "parse_json(jsonStr) - Parse a JSON string as a Variant value. Throw an exception when the string is not valid JSON value."
+  examples: |2-
+    
+        Examples:
+          > SELECT parse_json('{"a":1,"b":0.8}');
+           {"a":1,"b":0.8}
+      
+  since: "4.0.0"
+
+- name: "schema_of_variant"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.SchemaOfVariant"
+  signatures:
+    - "schema_of_variant(v)"
+  usage: "schema_of_variant(v) - Returns schema in the SQL format of a variant."
+  examples: |2-
+    
+        Examples:
+          > SELECT schema_of_variant(parse_json('null'));
+           VOID
+          > SELECT schema_of_variant(parse_json('[{"b":true,"a":0}]'));
+           ARRAY>
+      
+  since: "4.0.0"
+
+- name: "schema_of_variant_agg"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.SchemaOfVariantAgg"
+  signatures:
+    - "schema_of_variant_agg(v)"
+  usage: "schema_of_variant_agg(v) - Returns the merged schema in the SQL format of a variant column."
+  examples: |2-
+    
+        Examples:
+          > SELECT schema_of_variant_agg(parse_json(j)) FROM VALUES ('1'), ('2'), ('3') AS tab(j);
+           BIGINT
+          > SELECT schema_of_variant_agg(parse_json(j)) FROM VALUES ('{"a": 1}'), ('{"b": true}'), ('{"c": 1.23}') AS tab(j);
+           OBJECT
+      
+  since: "4.0.0"
+
+- name: "to_variant_object"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.ToVariantObject"
+  signatures:
+    - "to_variant_object(expr)"
+  usage: "to_variant_object(expr) - Convert a nested input (array/map/struct) into a variant where maps and structs are converted to variant objects which are unordered unlike SQL structs. Input maps can only have string keys."
+  examples: |2-
+    
+        Examples:
+          > SELECT to_variant_object(named_struct('a', 1, 'b', 2));
+           {"a":1,"b":2}
+          > SELECT to_variant_object(array(1, 2, 3));
+           [1,2,3]
+          > SELECT to_variant_object(array(named_struct('a', 1)));
+           [{"a":1}]
+          > SELECT to_variant_object(array(map("a", 2)));
+           [{"a":2}]
+      
+  since: "4.0.0"
+
+- name: "try_parse_json"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.TryParseJsonExpressionBuilder"
+  signatures:
+    - "try_parse_json(jsonStr)"
+  usage: "try_parse_json(jsonStr) - Parse a JSON string as a Variant value. Return NULL when the string is not valid JSON value."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_parse_json('{"a":1,"b":0.8}');
+           {"a":1,"b":0.8}
+          > SELECT try_parse_json('{"a":1,');
+           NULL
+      
+  since: "4.0.0"
+
+- name: "try_variant_get"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.TryVariantGetExpressionBuilder"
+  signatures:
+    - "try_variant_get(v, path[, type])"
+  usage: "try_variant_get(v, path[, type]) - Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to `type`. When `type` is omitted, it is default to `variant`. Returns null if the path does not exist or the cast fails."
+  examples: |2-
+    
+        Examples:
+          > SELECT try_variant_get(parse_json('{"a": 1}'), '$.a', 'int');
+           1
+          > SELECT try_variant_get(parse_json('{"a": 1}'), '$.b', 'int');
+           NULL
+          > SELECT try_variant_get(parse_json('[1, "2"]'), '$[1]', 'string');
+           2
+          > SELECT try_variant_get(parse_json('[1, "2"]'), '$[2]', 'string');
+           NULL
+          > SELECT try_variant_get(parse_json('[1, "hello"]'), '$[1]');
+           "hello"
+          > SELECT try_variant_get(parse_json('[1, "hello"]'), '$[1]', 'int');
+           NULL
+      
+  since: "4.0.0"
+
+- name: "variant_get"
+  scope: "sql_builtin"
+  group: "variant_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.variant.VariantGetExpressionBuilder"
+  signatures:
+    - "variant_get(v, path[, type])"
+  usage: "variant_get(v, path[, type]) - Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to `type`. When `type` is omitted, it is default to `variant`. Returns null if the path does not exist. Throws an exception if the cast fails."
+  examples: |2-
+    
+        Examples:
+          > SELECT variant_get(parse_json('{"a": 1}'), '$.a', 'int');
+           1
+          > SELECT variant_get(parse_json('{"a": 1}'), '$.b', 'int');
+           NULL
+          > SELECT variant_get(parse_json('[1, "2"]'), '$[1]', 'string');
+           2
+          > SELECT variant_get(parse_json('[1, "2"]'), '$[2]', 'string');
+           NULL
+          > SELECT variant_get(parse_json('[1, "hello"]'), '$[1]');
+           "hello"
+      
+  since: "4.0.0"
+
+- name: "variant_to_json"
+  scope: "sail_extension"
+  signatures:
+    - "variant_to_json(expr), variant_to_json(expr, options)"
+  usage: "variant_to_json(expr), variant_to_json(expr, options) - Returns a JSON string representation of a VARIANT expression."
diff --git a/crates/sail-plan/data/functions/scalar/vector.yaml b/crates/sail-plan/data/functions/scalar/vector.yaml
new file mode 100644
index 000000000..29f828532
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/vector.yaml
@@ -0,0 +1,109 @@
+- name: "vector_cosine_similarity"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorCosineSimilarity"
+  signatures:
+    - "vector_cosine_similarity(array1, array2)"
+  usage: |2-
+    
+        vector_cosine_similarity(array1, array2) - Returns the cosine similarity between two float vectors.
+        The vectors must have the same dimension.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_cosine_similarity(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));
+           0.9746319
+      
+  since: "4.2.0"
+
+- name: "vector_inner_product"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorInnerProduct"
+  signatures:
+    - "vector_inner_product(array1, array2)"
+  usage: |2-
+    
+        vector_inner_product(array1, array2) - Returns the inner product (dot product) between two float vectors.
+        The vectors must have the same dimension.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_inner_product(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));
+           32.0
+      
+  since: "4.2.0"
+
+- name: "vector_l2_distance"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorL2Distance"
+  signatures:
+    - "vector_l2_distance(array1, array2)"
+  usage: |2-
+    
+        vector_l2_distance(array1, array2) - Returns the Euclidean (L2) distance between two float vectors.
+        The vectors must have the same dimension.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_l2_distance(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));
+           5.196152
+      
+  since: "4.2.0"
+
+- name: "vector_norm"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorNorm"
+  signatures:
+    - "vector_norm(vector, degree)"
+  usage: |2-
+    
+        vector_norm(vector, degree) - Returns the Lp norm of a float vector using the specified degree.
+        Degree defaults to 2.0 (Euclidean norm) if unspecified. Supported values: 1.0 (L1 norm),
+        2.0 (L2 norm), float('inf') (infinity norm).
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_norm(array(3.0F, 4.0F), 2.0F);
+           5.0
+          > SELECT vector_norm(array(3.0F, 4.0F), 1.0F);
+           7.0
+          > SELECT vector_norm(array(3.0F, 4.0F), float('inf'));
+           4.0
+      
+  since: "4.2.0"
+
+- name: "vector_normalize"
+  scope: "sql_builtin"
+  group: "vector_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.VectorNormalize"
+  signatures:
+    - "vector_normalize(vector, degree)"
+  usage: |2-
+    
+        vector_normalize(vector, degree) - Normalizes a float vector to unit length using the specified norm degree.
+        Degree defaults to 2.0 (Euclidean norm) if unspecified. Supported values: 1.0 (L1 norm),
+        2.0 (L2 norm), float('inf') (infinity norm).
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT vector_normalize(array(3.0F, 4.0F), 2.0F);
+           [0.6,0.8]
+          > SELECT vector_normalize(array(3.0F, 4.0F), 1.0F);
+           [0.42857143,0.5714286]
+          > SELECT vector_normalize(array(3.0F, 4.0F), float('inf'));
+           [0.75,1.0]
+      
+  since: "4.2.0"
diff --git a/crates/sail-plan/data/functions/scalar/xml.yaml b/crates/sail-plan/data/functions/scalar/xml.yaml
new file mode 100644
index 000000000..01c9a9571
--- /dev/null
+++ b/crates/sail-plan/data/functions/scalar/xml.yaml
@@ -0,0 +1,207 @@
+- name: "from_xml"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.XmlToStructs"
+  signatures:
+    - "from_xml(xmlStr, schema[, options])"
+  usage: "from_xml(xmlStr, schema[, options]) - Returns a struct value with the given `xmlStr` and `schema`."
+  examples: |2-
+    
+        Examples:
+          > SELECT from_xml('
10.8
', 'a INT, b DOUBLE');
+           {"a":1,"b":0.8}
+          > SELECT from_xml('
', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
+           {"time":2015-08-26 00:00:00}
+          > SELECT from_xml('
AliceBob1Charlie2
', 'STRUCT>>');
+           {"teacher":"Alice","student":[{"name":"Bob","rank":1},{"name":"Charlie","rank":2}]}
+      
+  since: "4.0.0"
+
+- name: "schema_of_xml"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.SchemaOfXml"
+  signatures:
+    - "schema_of_xml(xml[, options])"
+  usage: "schema_of_xml(xml[, options]) - Returns schema in the DDL format of XML string."
+  examples: |2-
+    
+        Examples:
+          > SELECT schema_of_xml('
1
');
+           STRUCT
+          > SELECT schema_of_xml('
13
', map('excludeAttribute', 'true'));
+           STRUCT>
+      
+  since: "4.0.0"
+
+- name: "to_xml"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.StructsToXml"
+  signatures:
+    - "to_xml(expr[, options])"
+  usage: "to_xml(expr[, options]) - Returns a XML string with a given struct value"
+  examples: |2-
+    
+        Examples:
+          > SELECT to_xml(named_struct('a', 1, 'b', 2));
+           
+               1
+               2
+           
+          > SELECT to_xml(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));
+           
+               
+           
+      
+  since: "4.0.0"
+
+- name: "xpath"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathList"
+  signatures:
+    - "xpath(xml, xpath)"
+  usage: "xpath(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath('b1b2b3c1c2','a/b/text()');
+           ["b1","b2","b3"]
+          > SELECT xpath('b1b2b3c1c2','a/b');
+           [null,null,null]
+      
+  since: "2.0.0"
+
+- name: "xpath_boolean"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathBoolean"
+  signatures:
+    - "xpath_boolean(xml, xpath)"
+  usage: "xpath_boolean(xml, xpath) - Returns true if the XPath expression evaluates to true, or if a matching node is found."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_boolean('1','a/b');
+           true
+      
+  since: "2.0.0"
+
+- name: "xpath_double"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathDouble"
+  signatures:
+    - "xpath_double(xml, xpath)"
+  usage: "xpath_double(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_double('12', 'sum(a/b)');
+           3.0
+      
+  since: "2.0.0"
+
+- name: "xpath_float"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathFloat"
+  signatures:
+    - "xpath_float(xml, xpath)"
+  usage: "xpath_float(xml, xpath) - Returns a float value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_float('12', 'sum(a/b)');
+           3.0
+      
+  since: "2.0.0"
+
+- name: "xpath_int"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathInt"
+  signatures:
+    - "xpath_int(xml, xpath)"
+  usage: "xpath_int(xml, xpath) - Returns an integer value, or the value zero if no match is found, or a match is found but the value is non-numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_int('12', 'sum(a/b)');
+           3
+      
+  since: "2.0.0"
+
+- name: "xpath_long"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathLong"
+  signatures:
+    - "xpath_long(xml, xpath)"
+  usage: "xpath_long(xml, xpath) - Returns a long integer value, or the value zero if no match is found, or a match is found but the value is non-numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_long('12', 'sum(a/b)');
+           3
+      
+  since: "2.0.0"
+
+- name: "xpath_number"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathDouble"
+  signatures:
+    - "xpath_number(xml, xpath)"
+  usage: "xpath_number(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_number('12', 'sum(a/b)');
+           3.0
+      
+  since: "2.0.0"
+
+- name: "xpath_short"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathShort"
+  signatures:
+    - "xpath_short(xml, xpath)"
+  usage: "xpath_short(xml, xpath) - Returns a short integer value, or the value zero if no match is found, or a match is found but the value is non-numeric."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_short('12', 'sum(a/b)');
+           3
+      
+  since: "2.0.0"
+
+- name: "xpath_string"
+  scope: "sql_builtin"
+  group: "xml_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.xml.XPathString"
+  signatures:
+    - "xpath_string(xml, xpath)"
+  usage: "xpath_string(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression."
+  examples: |2-
+    
+        Examples:
+          > SELECT xpath_string('bcc','a/c');
+           cc
+      
+  since: "2.0.0"
+
diff --git a/crates/sail-plan/data/functions/table/range.yaml b/crates/sail-plan/data/functions/table/range.yaml
new file mode 100644
index 000000000..a183cc055
--- /dev/null
+++ b/crates/sail-plan/data/functions/table/range.yaml
@@ -0,0 +1,45 @@
+- name: "range"
+  scope: "sql_builtin"
+  group: "table_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.plans.logical.Range"
+  signatures:
+    - "range(start[, end[, step[, numSlices]]]) / range(end)"
+  usage: |2-
+    
+        range(start[, end[, step[, numSlices]]]) / range(end) - Returns a table of values within a specified range.
+      
+  arguments: |2-
+    
+        Arguments:
+          * start - An optional BIGINT literal defaulted to 0, marking the first value generated.
+          * end - A BIGINT literal marking endpoint (exclusive) of the number generation.
+          * step - An optional BIGINT literal defaulted to 1, specifying the increment used when generating values.
+          * numParts - An optional INTEGER literal specifying how the production of rows is spread across partitions.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT * FROM range(1);
+            +---+
+            | id|
+            +---+
+            |  0|
+            +---+
+          > SELECT * FROM range(0, 2);
+            +---+
+            |id |
+            +---+
+            |0  |
+            |1  |
+            +---+
+          > SELECT * FROM range(0, 4, 2);
+            +---+
+            |id |
+            +---+
+            |0  |
+            |2  |
+            +---+
+      
+  since: "2.0.0"
+
diff --git a/crates/sail-plan/data/functions/window.yaml b/crates/sail-plan/data/functions/window.yaml
new file mode 100644
index 000000000..781d795c9
--- /dev/null
+++ b/crates/sail-plan/data/functions/window.yaml
@@ -0,0 +1,267 @@
+- name: "cume_dist"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.CumeDist"
+  signatures:
+    - "cume_dist()"
+  usage: |2-
+    
+        cume_dist() - Computes the position of a value relative to all values in the partition.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	0.6666666666666666
+           A1	1	0.6666666666666666
+           A1	2	1.0
+           A2	3	1.0
+      
+  since: "2.0.0"
+
+- name: "dense_rank"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.DenseRank"
+  signatures:
+    - "dense_rank()"
+  usage: |2-
+    
+        dense_rank() - Computes the rank of a value in a group of values. The result is one plus the
+          previously assigned rank value. Unlike the function rank, dense_rank will not produce gaps
+          in the ranking sequence.
+      
+  arguments: |2-
+    
+        Arguments:
+          * children - this is to base the rank on; a change in the value of one the children will
+              trigger a change in rank. This is an internal parameter and will be assigned by the
+              Analyser.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	1
+           A1	1	1
+           A1	2	2
+           A2	3	1
+      
+  since: "2.0.0"
+
+- name: "lag"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Lag"
+  signatures:
+    - "lag(input[, offset[, default]])"
+  usage: |2-
+    
+        lag(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row
+          before the current row in the window. The default value of `offset` is 1 and the default
+          value of `default` is null. If the value of `input` at the `offset`th row is null,
+          null is returned. If there is no such offset row (e.g., when the offset is 1, the first
+          row of the window does not have any previous row), `default` is returned.
+      
+  arguments: |2-
+    
+        Arguments:
+          * input - a string expression to evaluate `offset` rows before the current row.
+          * offset - an int expression which is rows to jump back in the partition.
+          * default - a string expression which is to use when the offset row does not exist.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	NULL
+           A1	1	1
+           A1	2	1
+           A2	3	NULL
+      
+  since: "2.0.0"
+
+- name: "lead"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Lead"
+  signatures:
+    - "lead(input[, offset[, default]])"
+  usage: |2-
+    
+        lead(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row
+          after the current row in the window. The default value of `offset` is 1 and the default
+          value of `default` is null. If the value of `input` at the `offset`th row is null,
+          null is returned. If there is no such an offset row (e.g., when the offset is 1, the last
+          row of the window does not have any subsequent row), `default` is returned.
+      
+  arguments: |2-
+    
+        Arguments:
+          * input - a string expression to evaluate `offset` rows after the current row.
+          * offset - an int expression which is rows to jump ahead in the partition.
+          * default - a string expression which is to use when the offset is larger than the window.
+              The default value is null.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	1
+           A1	1	2
+           A1	2	NULL
+           A2	3	NULL
+      
+  since: "2.0.0"
+
+- name: "nth_value"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.NthValue"
+  signatures:
+    - "nth_value(input[, offset])"
+  usage: |2-
+    
+        nth_value(input[, offset]) - Returns the value of `input` at the row that is the `offset`th row
+          from beginning of the window frame. Offset starts at 1. If ignoreNulls=true, we will skip
+          nulls when finding the `offset`th row. Otherwise, every row counts for the `offset`. If
+          there is no such an `offset`th row (e.g., when the offset is 10, size of the window frame
+          is less than 10), null is returned.
+      
+  arguments: |2-
+    
+        Arguments:
+          * input - the target column or expression that the function operates on.
+          * offset - a positive int literal to indicate the offset in the window frame. It starts
+              with 1.
+          * ignoreNulls - an optional specification that indicates the NthValue should skip null
+              values in the determination of which row to use.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	1
+           A1	1	1
+           A1	2	1
+           A2	3	NULL
+      
+  since: "3.1.0"
+
+- name: "ntile"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.NTile"
+  signatures:
+    - "ntile(n)"
+  usage: |2-
+    
+        ntile(n) - Divides the rows for each window partition into `n` buckets ranging
+          from 1 to at most `n`.
+      
+  arguments: |2-
+    
+        Arguments:
+          * buckets - an int expression which is number of buckets to divide the rows in.
+              Default value is 1.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	1
+           A1	1	1
+           A1	2	2
+           A2	3	1
+      
+  since: "2.0.0"
+
+- name: "percent_rank"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.PercentRank"
+  signatures:
+    - "percent_rank()"
+  usage: |2-
+    
+        percent_rank() - Computes the percentage ranking of a value in a group of values.
+      
+  arguments: |2-
+    
+        Arguments:
+          * children - this is to base the rank on; a change in the value of one the children will
+              trigger a change in rank. This is an internal parameter and will be assigned by the
+              Analyser.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	0.0
+           A1	1	0.0
+           A1	2	1.0
+           A2	3	0.0
+      
+  since: "2.0.0"
+
+- name: "rank"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.Rank"
+  signatures:
+    - "rank()"
+  usage: |2-
+    
+        rank() - Computes the rank of a value in a group of values. The result is one plus the number
+          of rows preceding or equal to the current row in the ordering of the partition. The values
+          will produce gaps in the sequence.
+      
+  arguments: |2-
+    
+        Arguments:
+          * children - this is to base the rank on; a change in the value of one the children will
+              trigger a change in rank. This is an internal parameter and will be assigned by the
+              Analyser.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	1
+           A1	1	1
+           A1	2	3
+           A2	3	1
+      
+  since: "2.0.0"
+
+- name: "row_number"
+  scope: "sql_builtin"
+  group: "window_funcs"
+  source: "built-in"
+  class_name: "org.apache.spark.sql.catalyst.expressions.RowNumber"
+  signatures:
+    - "row_number()"
+  usage: |2-
+    
+        row_number() - Assigns a unique, sequential number to each row, starting with one,
+          according to the ordering of rows within the window partition.
+      
+  examples: |2-
+    
+        Examples:
+          > SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);
+           A1	1	1
+           A1	1	2
+           A1	2	3
+           A2	3	1
+      
+  since: "2.0.0"
+
diff --git a/crates/sail-plan/src/catalog.rs b/crates/sail-plan/src/catalog.rs
index f2f1508ee..2aae31258 100644
--- a/crates/sail-plan/src/catalog.rs
+++ b/crates/sail-plan/src/catalog.rs
@@ -3,12 +3,14 @@ use std::fmt::Formatter;
 use datafusion::common::{DFSchemaRef, Result};
 use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
 use datafusion::prelude::SessionContext;
-use datafusion_common::{internal_datafusion_err, DFSchema};
+use datafusion_common::{DFSchema, internal_datafusion_err};
 use educe::Educe;
 use sail_catalog::command::CatalogCommand;
 use sail_catalog::utils::quote_names_if_needed;
 use sail_common_datafusion::catalog::display::CatalogObjectDisplay;
-use sail_common_datafusion::catalog::{DatabaseStatus, TableColumnStatus, TableStatus};
+use sail_common_datafusion::catalog::{
+    DatabaseStatus, FunctionStatus, TableColumnStatus, TableStatus,
+};
 use sail_common_datafusion::session::plan::PlanFormatter;
 use sail_common_datafusion::utils::items::ItemTaker;
 
@@ -177,14 +179,15 @@ impl CatalogObjectDisplay for SparkCatalogObjectDisplay {
         }
     }
 
-    fn function(name: String) -> Self::Function {
+    fn function(status: FunctionStatus) -> Self::Function {
+        let description = status.list_description();
         Self::Function {
-            name,
-            catalog: None,
-            namespace: None,
-            description: None,
-            class_name: "".to_string(),
-            is_temporary: false,
+            name: status.name,
+            catalog: status.catalog,
+            namespace: status.namespace,
+            description,
+            class_name: status.class_name,
+            is_temporary: status.is_temporary,
         }
     }
 }
diff --git a/crates/sail-plan/src/config.rs b/crates/sail-plan/src/config.rs
index 9a8f0235e..e5eca020a 100644
--- a/crates/sail-plan/src/config.rs
+++ b/crates/sail-plan/src/config.rs
@@ -16,6 +16,8 @@ pub enum DefaultTimestampType {
 pub struct PlanConfig {
     /// The time zone of the session.
     pub session_timezone: Arc,
+    /// The locale of the session.
+    pub session_locale: Arc,
     /// The default timestamp type.
     pub default_timestamp_type: DefaultTimestampType,
     /// Whether to use large variable types in Arrow.
@@ -36,6 +38,10 @@ pub struct PlanConfig {
     /// The maximum number of distinct values collected for a pivot without an explicit
     /// value list (`spark.sql.pivotMaxValues`, default 10000). Exceeding it is an error.
     pub pivot_max_values: usize,
+    /// Whether a table-valued function may receive more than one `TABLE (...)` argument
+    /// (`spark.sql.tvf.allowMultipleTableArguments.enabled`, default false). Multiple table
+    /// arguments produce the cartesian product of their rows.
+    pub tvf_allow_multiple_table_arguments: bool,
 }
 
 impl PlanConfig {
@@ -51,6 +57,7 @@ impl Default for PlanConfig {
     fn default() -> Self {
         Self {
             session_timezone: Arc::from("UTC"),
+            session_locale: Arc::from("en-US"),
             default_timestamp_type: DefaultTimestampType::TimestampLtz,
             arrow_use_large_var_types: false,
             pyspark_udf_config: Arc::new(PySparkUdfConfig::default()),
@@ -61,6 +68,7 @@ impl Default for PlanConfig {
             cross_join_enabled: true,
             case_sensitive: false,
             pivot_max_values: 10000,
+            tvf_allow_multiple_table_arguments: false,
         }
     }
 }
diff --git a/crates/sail-plan/src/explain.rs b/crates/sail-plan/src/explain.rs
index 5e7993b7e..a7a7630ac 100644
--- a/crates/sail-plan/src/explain.rs
+++ b/crates/sail-plan/src/explain.rs
@@ -3,18 +3,20 @@ use std::sync::Arc;
 
 use datafusion::execution::session_state::SessionStateBuilder;
 use datafusion::physical_plan::display::DisplayableExecutionPlan;
-use datafusion::physical_plan::{collect, displayable, ExecutionPlan};
+use datafusion::physical_plan::{ExecutionPlan, collect, displayable};
 use datafusion::prelude::SessionContext;
 use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan};
 use datafusion_common::{DataFusionError, Result};
 use datafusion_expr::LogicalPlan;
 use sail_common::spec;
+use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::rename::physical_plan::rename_physical_plan;
+use sail_common_datafusion::session::job::JobService;
 
 use crate::config::PlanConfig;
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::plan::NamedPlan;
 use crate::resolver::PlanResolver;
+use crate::resolver::plan::NamedPlan;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ExplainKind {
@@ -369,6 +371,18 @@ async fn maybe_collect_metrics(
     Ok(())
 }
 
+fn distributed_plan_string(
+    ctx: &SessionContext,
+    physical: &Option>,
+) -> Option {
+    let plan = physical.as_ref()?;
+    let service = ctx.extension::().ok()?;
+    match service.runner().explain(Arc::clone(plan)) {
+        Ok(plan) => Some(plan),
+        Err(err) => Some(format!("Distributed plan error: {err}")),
+    }
+}
+
 pub async fn explain_string(
     ctx: &SessionContext,
     config: Arc,
@@ -414,7 +428,7 @@ async fn explain_from_collected(
 
     let mut physical = PhysicalStrings::default();
 
-    let output = match options.kind {
+    let sections = match options.kind {
         ExplainKind::Simple => {
             let physical_for_mode = if options.analyze {
                 physical.full_with_metrics(&collected)
@@ -432,59 +446,69 @@ async fn explain_from_collected(
                     physical.with_schema(&collected),
                 ));
             }
-            sections.join("\n\n")
+            sections
+        }
+        ExplainKind::Extended => {
+            vec![
+                render_section("Parsed Logical Plan", &logical_simple),
+                // prepend schema to make analyzed plan distinct from optimized plan
+                render_section("Analyzed Logical Plan", &logical_analyzed_schema),
+                render_section("Optimized Logical Plan", &logical_optimized),
+                render_section(
+                    "Physical Plan",
+                    if options.analyze {
+                        physical.full_with_metrics(&collected)
+                    } else {
+                        physical.plain(&collected, options.verbose)
+                    },
+                ),
+            ]
+        }
+        ExplainKind::Codegen => {
+            let mut sections = vec![
+                render_section(
+                    "Codegen",
+                    "Whole-stage codegen is not supported; showing physical plan instead.",
+                ),
+                render_section(
+                    "Plan Steps",
+                    &render_stringified_plans(&collected.stringified),
+                ),
+                render_section(
+                    "Physical Plan",
+                    if options.analyze {
+                        physical.full_with_metrics(&collected)
+                    } else {
+                        physical.plain(&collected, options.verbose)
+                    },
+                ),
+            ];
+            if let Some(plan) = distributed_plan_string(ctx, &collected.physical_plan) {
+                sections.push(render_section("Distributed Plan", &plan));
+            }
+            sections
+        }
+        ExplainKind::Cost => {
+            vec![
+                render_section("Optimized Logical Plan", &logical_optimized),
+                // TODO: Spark COST mode shows logical plan + stats; we currently return physical +
+                // stats
+                render_section(
+                    "Physical Plan",
+                    if options.verbose || options.analyze {
+                        physical.full(&collected)
+                    } else {
+                        physical.with_stats(&collected)
+                    },
+                ),
+            ]
         }
-        ExplainKind::Extended => [
-            render_section("Parsed Logical Plan", &logical_simple),
-            // prepend schema to make analyzed plan distinct from optimized plan
-            render_section("Analyzed Logical Plan", &logical_analyzed_schema),
-            render_section("Optimized Logical Plan", &logical_optimized),
-            render_section(
-                "Physical Plan",
-                if options.analyze {
-                    physical.full_with_metrics(&collected)
-                } else {
-                    physical.plain(&collected, options.verbose)
-                },
-            ),
-        ]
-        .join("\n\n"),
-        ExplainKind::Codegen => [
-            render_section(
-                "Codegen",
-                "Whole-stage codegen is not supported; showing physical plan instead.",
-            ),
-            render_section(
-                "Plan Steps",
-                &render_stringified_plans(&collected.stringified),
-            ),
-            render_section(
-                "Physical Plan",
-                if options.analyze {
-                    physical.full_with_metrics(&collected)
-                } else {
-                    physical.plain(&collected, options.verbose)
-                },
-            ),
-        ]
-        .join("\n\n"),
-        ExplainKind::Cost => [
-            render_section("Optimized Logical Plan", &logical_optimized),
-            // TODO: Spark COST mode shows logical plan + stats; we currently return physical +
-            // stats
-            render_section(
-                "Physical Plan",
-                if options.verbose || options.analyze {
-                    physical.full(&collected)
-                } else {
-                    physical.with_stats(&collected)
-                },
-            ),
-        ]
-        .join("\n\n"),
         // TODO: Spark FORMATTED mode emits outline + node details
-        ExplainKind::Formatted => render_section("Physical Plan", physical.full(&collected)),
+        ExplainKind::Formatted => {
+            vec![render_section("Physical Plan", physical.full(&collected))]
+        }
     };
+    let output = sections.join("\n\n");
 
     Ok(ExplainString {
         output,
diff --git a/crates/sail-plan/src/formatter.rs b/crates/sail-plan/src/formatter.rs
index debb23e2c..0b31ca126 100644
--- a/crates/sail-plan/src/formatter.rs
+++ b/crates/sail-plan/src/formatter.rs
@@ -1,10 +1,10 @@
 use std::fmt::{Debug, Display, Formatter};
 use std::hash::Hash;
 
-use datafusion::arrow::array::timezone::Tz;
 use datafusion::arrow::array::Array;
+use datafusion::arrow::array::timezone::Tz;
 use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit};
-use datafusion_common::{not_impl_err, plan_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, not_impl_err, plan_err};
 use half::f16;
 use sail_common_datafusion::formatter::{
     Date32Formatter, Date64Formatter, DurationMicrosecondFormatter, DurationMillisecondFormatter,
@@ -775,7 +775,7 @@ mod tests {
     };
     use datafusion::arrow::buffer::OffsetBuffer;
     use datafusion::arrow::datatypes::{
-        i256, DataType, Field, Int32Type, IntervalDayTime, IntervalMonthDayNano,
+        DataType, Field, Int32Type, IntervalDayTime, IntervalMonthDayNano, i256,
     };
     use datafusion_common::arrow::array::ArrayRef;
     use sail_common::spec::SAIL_MAP_FIELD_NAME;
diff --git a/crates/sail-plan/src/function/aggregate.rs b/crates/sail-plan/src/function/aggregate.rs
index e863663db..89d0180c0 100644
--- a/crates/sail-plan/src/function/aggregate.rs
+++ b/crates/sail-plan/src/function/aggregate.rs
@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use arrow::datatypes::{DataType, Field};
+use datafusion::functions::expr_fn::coalesce;
 use datafusion::functions_aggregate::{
     approx_distinct, approx_percentile_cont, array_agg, average, bit_and_or_xor, bool_and_or,
     correlation, count, covariance, first_last, grouping, min_max, percentile_cont, stddev, sum,
@@ -10,7 +11,9 @@ use datafusion::functions_aggregate::{
 use datafusion::functions_nested::string::array_to_string;
 use datafusion_common::ScalarValue;
 use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams};
-use datafusion_expr::{cast, expr, lit, when, AggregateUDF, ExprSchemable, ScalarUDF};
+use datafusion_expr::{
+    AggregateUDF, BinaryExpr, ExprSchemable, Operator, ScalarUDF, cast, expr, lit, when,
+};
 use datafusion_spark::function::aggregate::try_sum::SparkTrySum;
 use lazy_static::lazy_static;
 use sail_common::spec::SAIL_LIST_FIELD_NAME;
@@ -39,9 +42,9 @@ use sail_function::scalar::struct_function::StructFunction;
 
 use crate::error::{PlanError, PlanResult};
 use crate::function::common::{
-    count_min_sketch_args, get_arguments_and_null_treatment, get_null_treatment,
-    hll_args_with_default_lg, hll_union_args_with_default_allow_different_lg,
-    theta_args_with_default_lg, AggFunction, AggFunctionInput,
+    AggFunction, AggFunctionInput, count_min_sketch_args, get_arguments_and_null_treatment,
+    get_null_treatment, hll_args_with_default_lg, hll_union_args_with_default_allow_different_lg,
+    theta_args_with_default_lg,
 };
 use crate::function::transform_count_star_wildcard_expr;
 
@@ -73,6 +76,100 @@ fn avg(input: AggFunctionInput) -> PlanResult {
     }))
 }
 
+fn spark_sum(input: AggFunctionInput) -> PlanResult {
+    let AggFunctionInput {
+        arguments,
+        distinct,
+        ignore_nulls,
+        filter,
+        order_by,
+        function_context,
+    } = input;
+    let supports_linear_rewrite =
+        !distinct && ignore_nulls.is_none() && filter.is_none() && order_by.is_empty();
+    let arguments = if supports_linear_rewrite {
+        arguments
+            .into_iter()
+            .map(|argument| {
+                widen_safe_linear_sum_argument(argument, function_context.schema.as_ref())
+            })
+            .collect()
+    } else {
+        arguments
+    };
+
+    Ok(expr::Expr::AggregateFunction(AggregateFunction {
+        func: sum::sum_udaf(),
+        params: AggregateFunctionParams {
+            args: arguments,
+            distinct,
+            filter,
+            order_by,
+            null_treatment: get_null_treatment(ignore_nulls),
+        },
+    }))
+}
+
+fn widen_safe_linear_sum_argument(
+    argument: expr::Expr,
+    schema: &datafusion_common::DFSchema,
+) -> expr::Expr {
+    if argument.get_type(schema).ok() != Some(DataType::Int32) {
+        return argument;
+    }
+    let expr::Expr::BinaryExpr(BinaryExpr { left, op, right }) = &argument else {
+        return argument;
+    };
+    if *op != Operator::Plus {
+        return argument;
+    }
+
+    let left_literal = widen_int32_literal(left);
+    let right_literal = widen_int32_literal(right);
+    let (value, widened_literal, value_expression, literal_is_left) =
+        match (left_literal, right_literal) {
+            (Some((value, literal)), None) => (value, literal, right.as_ref(), true),
+            (None, Some((value, literal))) => (value, literal, left.as_ref(), false),
+            _ => return argument,
+        };
+    let Ok(value_type) = value_expression.get_type(schema) else {
+        return argument;
+    };
+    let Some((minimum, maximum)) = signed_integer_bounds(&value_type) else {
+        return argument;
+    };
+    if minimum + value < i64::from(i32::MIN) || maximum + value > i64::from(i32::MAX) {
+        return argument;
+    }
+
+    let widened_value = cast(value_expression.clone(), DataType::Int64);
+    if literal_is_left {
+        widened_literal + widened_value
+    } else {
+        widened_value + widened_literal
+    }
+}
+
+fn widen_int32_literal(argument: &expr::Expr) -> Option<(i64, expr::Expr)> {
+    let expr::Expr::Literal(ScalarValue::Int32(Some(value)), metadata) = argument else {
+        return None;
+    };
+    let value = i64::from(*value);
+    Some((
+        value,
+        expr::Expr::Literal(ScalarValue::Int64(Some(value)), metadata.clone()),
+    ))
+}
+
+fn signed_integer_bounds(data_type: &DataType) -> Option<(i64, i64)> {
+    match data_type {
+        DataType::Int8 => Some((i64::from(i8::MIN), i64::from(i8::MAX))),
+        DataType::Int16 => Some((i64::from(i16::MIN), i64::from(i16::MAX))),
+        DataType::Int32 => Some((i64::from(i32::MIN), i64::from(i32::MAX))),
+        _ => None,
+    }
+}
+
 fn grouping_id(input: AggFunctionInput) -> PlanResult {
     let AggFunctionInput {
         arguments,
@@ -104,7 +201,8 @@ fn first_value(input: AggFunctionInput) -> PlanResult {
         func: first_last::first_value_udaf(),
         params: AggregateFunctionParams {
             args,
-            distinct: input.distinct,
+            // Spark's `EliminateDistinct` strips DISTINCT from `First` as duplicate-agnostic.
+            distinct: false,
             filter: input.filter,
             order_by: input.order_by,
             null_treatment,
@@ -119,7 +217,8 @@ fn last_value(input: AggFunctionInput) -> PlanResult {
         func: first_last::last_value_udaf(),
         params: AggregateFunctionParams {
             args,
-            distinct: input.distinct,
+            // Spark's `EliminateDistinct` strips DISTINCT from `Last` as duplicate-agnostic.
+            distinct: false,
             filter: input.filter,
             order_by: input.order_by,
             null_treatment,
@@ -249,9 +348,10 @@ fn percentile_disc(input: AggFunctionInput) -> PlanResult {
     let column = sort.expr;
     let percentile = input.arguments.one()?;
     let args = vec![column, percentile];
+    let ansi_mode = input.function_context.plan_config.ansi_mode;
 
     Ok(expr::Expr::AggregateFunction(AggregateFunction {
-        func: percentile_disc_udaf(),
+        func: percentile_disc_udaf(ansi_mode),
         params: AggregateFunctionParams {
             args,
             distinct: input.distinct,
@@ -387,71 +487,79 @@ fn count_if(input: AggFunctionInput) -> PlanResult {
 }
 
 fn collect_set(input: AggFunctionInput) -> PlanResult {
+    let schema = input.function_context.schema;
     // Spark's collect_set ignores NULLs by default
     let ignore_nulls = input.ignore_nulls.or(Some(true));
+    let arg = input.arguments.one()?;
+    let element_type = arg.get_type(schema)?;
 
     // WORKAROUND: DataFusion's array_agg doesn't properly handle null_treatment when distinct=true
     // So we need to add an explicit filter for NULLs
-    let (args, filter, null_treatment) = if ignore_nulls == Some(true) {
-        let arg = input.arguments.one()?;
+    let (filter, null_treatment) = if ignore_nulls == Some(true) {
         let null_filter = arg.clone().is_not_null();
         let combined_filter = match input.filter {
-            Some(existing) => Some(Box::new(existing.as_ref().clone().and(null_filter))),
-            None => Some(Box::new(null_filter)),
+            Some(existing) => existing.as_ref().clone().and(null_filter),
+            None => null_filter,
         };
-        (vec![arg], combined_filter, None) // Don't use null_treatment when we have explicit filter
+        // Don't use null_treatment when we have explicit filter
+        (Some(Box::new(combined_filter)), None)
     } else {
-        (
-            input.arguments,
-            input.filter,
-            get_null_treatment(ignore_nulls),
-        )
+        (input.filter, get_null_treatment(ignore_nulls))
     };
 
-    Ok(expr::Expr::AggregateFunction(AggregateFunction {
+    let agg = expr::Expr::AggregateFunction(AggregateFunction {
         func: array_agg::array_agg_udaf(),
         params: AggregateFunctionParams {
-            args,
+            args: vec![arg],
             distinct: true,
             order_by: input.order_by,
             filter,
             null_treatment,
         },
-    }))
+    });
+    Ok(coalesce_to_empty_array(agg, &element_type))
 }
 
 fn array_agg_compacted(input: AggFunctionInput) -> PlanResult {
+    let schema = input.function_context.schema;
     // Spark's collect_list ignores NULLs by default
     let ignore_nulls = input.ignore_nulls.or(Some(true));
+    let arg = input.arguments.one()?;
+    let element_type = arg.get_type(schema)?;
 
     // WORKAROUND: DataFusion's array_agg doesn't properly handle null_treatment when distinct=true
     // So we need to add an explicit filter for NULLs when both distinct and ignore_nulls are true
-    let (args, filter, null_treatment) = if input.distinct && ignore_nulls == Some(true) {
-        let arg = input.arguments.one()?;
+    let (filter, null_treatment) = if input.distinct && ignore_nulls == Some(true) {
         let null_filter = arg.clone().is_not_null();
         let combined_filter = match input.filter {
-            Some(existing) => Some(Box::new(existing.as_ref().clone().and(null_filter))),
-            None => Some(Box::new(null_filter)),
+            Some(existing) => existing.as_ref().clone().and(null_filter),
+            None => null_filter,
         };
-        (vec![arg], combined_filter, None) // Don't use null_treatment when we have explicit filter
+        // Don't use null_treatment when we have explicit filter
+        (Some(Box::new(combined_filter)), None)
     } else {
-        (
-            input.arguments,
-            input.filter,
-            get_null_treatment(ignore_nulls),
-        )
+        (input.filter, get_null_treatment(ignore_nulls))
     };
 
-    Ok(expr::Expr::AggregateFunction(AggregateFunction {
+    let agg = expr::Expr::AggregateFunction(AggregateFunction {
         func: array_agg::array_agg_udaf(),
         params: AggregateFunctionParams {
-            args,
+            args: vec![arg],
             distinct: input.distinct,
             order_by: input.order_by,
             filter,
             null_treatment,
         },
-    }))
+    });
+    Ok(coalesce_to_empty_array(agg, &element_type))
+}
+
+/// Spark's `collect_list`/`collect_set` return an empty array for a group with no (non-NULL)
+/// values to collect, but DataFusion's `array_agg` returns NULL over zero rows. Coalesce the
+/// aggregate to a typed empty array so the empty case matches Spark instead of yielding NULL.
+fn coalesce_to_empty_array(agg: expr::Expr, element_type: &DataType) -> expr::Expr {
+    let empty = ScalarValue::List(ScalarValue::new_list_nullable(&[], element_type));
+    coalesce(vec![agg, lit(empty)])
 }
 
 fn listagg(input: AggFunctionInput) -> PlanResult {
@@ -676,6 +784,9 @@ fn list_built_in_aggregate_functions() -> Vec<(&'static str, AggFunction)> {
         ("histogram_numeric", F::custom(histogram_numeric)),
         ("hll_sketch_agg", F::custom(hll_sketch_agg)),
         ("hll_union_agg", F::custom(hll_union_agg)),
+        ("kll_merge_agg_bigint", F::unknown("kll_merge_agg_bigint")),
+        ("kll_merge_agg_double", F::unknown("kll_merge_agg_double")),
+        ("kll_merge_agg_float", F::unknown("kll_merge_agg_float")),
         (
             "theta_intersection_agg",
             F::default(|| Arc::new(AggregateUDF::from(ThetaIntersectionAggFunction::new()))),
@@ -689,6 +800,7 @@ fn list_built_in_aggregate_functions() -> Vec<(&'static str, AggFunction)> {
         ("max", F::default(min_max::max_udaf)),
         ("max_by", F::custom(max_by)),
         ("mean", F::default(average::avg_udaf)),
+        ("measure", F::unknown("measure")),
         ("median", F::custom(median)),
         ("min", F::default(min_max::min_udaf)),
         ("min_by", F::custom(min_by)),
@@ -750,12 +862,38 @@ fn list_built_in_aggregate_functions() -> Vec<(&'static str, AggFunction)> {
         ("stddev_pop", F::default(stddev::stddev_pop_udaf)),
         ("stddev_samp", F::default(stddev::stddev_udaf)),
         ("string_agg", F::custom(listagg)),
-        ("sum", F::default(sum::sum_udaf)),
+        ("sum", F::custom(spark_sum)),
+        (
+            "tuple_intersection_agg_double",
+            F::unknown("tuple_intersection_agg_double"),
+        ),
+        (
+            "tuple_intersection_agg_integer",
+            F::unknown("tuple_intersection_agg_integer"),
+        ),
+        (
+            "tuple_sketch_agg_double",
+            F::unknown("tuple_sketch_agg_double"),
+        ),
+        (
+            "tuple_sketch_agg_integer",
+            F::unknown("tuple_sketch_agg_integer"),
+        ),
+        (
+            "tuple_union_agg_double",
+            F::unknown("tuple_union_agg_double"),
+        ),
+        (
+            "tuple_union_agg_integer",
+            F::unknown("tuple_union_agg_integer"),
+        ),
         ("try_avg", F::custom(try_avg)),
         ("try_sum", F::custom(try_sum)),
         ("var_pop", F::default(variance::var_pop_udaf)),
         ("var_samp", F::default(variance::var_samp_udaf)),
         ("variance", F::default(variance::var_samp_udaf)),
+        ("vector_avg", F::unknown("vector_avg")),
+        ("vector_sum", F::unknown("vector_sum")),
     ]
 }
 
@@ -765,3 +903,7 @@ pub(crate) fn get_built_in_aggregate_function(name: &str) -> PlanResult impl Iterator {
+    BUILT_IN_AGGREGATE_FUNCTIONS.keys().copied()
+}
diff --git a/crates/sail-plan/src/function/common.rs b/crates/sail-plan/src/function/common.rs
index 5347da5ff..a686793e4 100644
--- a/crates/sail-plan/src/function/common.rs
+++ b/crates/sail-plan/src/function/common.rs
@@ -6,8 +6,8 @@ use datafusion::prelude::SessionContext;
 use datafusion_common::{DFSchemaRef, ScalarValue};
 use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams, WindowFunctionParams};
 use datafusion_expr::{
-    cast, expr, lit, AggregateUDF, BinaryExpr, ExprSchemable, Operator, ScalarUDF, ScalarUDFImpl,
-    WindowFrame, WindowFunctionDefinition, WindowUDF,
+    AggregateUDF, BinaryExpr, ExprSchemable, Operator, ScalarUDF, ScalarUDFImpl, WindowFrame,
+    WindowFunctionDefinition, WindowUDF, cast, expr, lit,
 };
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::sketch::{DEFAULT_HLL_LG_CONFIG_K, DEFAULT_THETA_LG_NOM_ENTRIES};
@@ -262,14 +262,9 @@ impl AggFunctionBuilder {
         Arc::new(f)
     }
 
-    #[expect(dead_code)]
     pub fn unknown(name: &str) -> AggFunction {
         let name = name.to_string();
-        Arc::new(move |_| {
-            Err(PlanError::todo(format!(
-                "unknown aggregate function: {name}"
-            )))
-        })
+        Arc::new(move |_| Err(PlanError::todo(format!("function: {name}"))))
     }
 }
 
@@ -398,7 +393,7 @@ pub(crate) fn get_arguments_and_null_treatment(
             _ => {
                 return Err(PlanError::invalid(
                     "requires a boolean literal as the second argument",
-                ))
+                ));
             }
         };
         Ok((vec![expr], null_treatment))
@@ -451,12 +446,11 @@ pub(crate) fn count_min_sketch_args(arguments: Vec) -> PlanResult {
             let (value, eps, confidence, seed) = arguments.four()?;
-            Ok(vec![
-                value,
-                cast(eps, DataType::Float64),
-                cast(confidence, DataType::Float64),
-                seed,
-            ])
+            // Spark requires `eps` and `confidence` to be DOUBLE literals and rejects
+            // other types (including DECIMAL/FLOAT). Pass them through unchanged so the
+            // aggregate's type validation enforces the same rule, instead of casting and
+            // silently accepting types Spark rejects.
+            Ok(vec![value, eps, confidence, seed])
         }
         count => Err(PlanError::invalid(format!(
             "count_min_sketch requires 4 arguments, got {count}"
diff --git a/crates/sail-plan/src/function/generator.rs b/crates/sail-plan/src/function/generator.rs
index 86f332dc2..890ad6ed5 100644
--- a/crates/sail-plan/src/function/generator.rs
+++ b/crates/sail-plan/src/function/generator.rs
@@ -1,15 +1,15 @@
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{lit, Expr, ScalarUDF};
+use datafusion_expr::{Expr, ScalarUDF, lit};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::array::arrays_zip::ArraysZip;
 use sail_function::scalar::array::spark_array::SparkArray;
 use sail_function::scalar::explode::{Explode, ExplodeKind};
 use sail_function::scalar::variant::spark_variant_explode::SparkVariantExplodeUdf;
 
+use crate::PlanResult;
 use crate::error::PlanError;
 use crate::function::common::{ScalarFunction, ScalarFunctionInput};
-use crate::PlanResult;
 
 fn stack(input: ScalarFunctionInput) -> PlanResult {
     let ScalarFunctionInput {
@@ -38,7 +38,7 @@ fn stack(input: ScalarFunctionInput) -> PlanResult {
         wrong_value => {
             return Err(PlanError::invalid(format!(
                 "stack expects first argument to be between (0, INT32::MAX], got {wrong_value}"
-            )))
+            )));
         }
     };
 
diff --git a/crates/sail-plan/src/function/metadata.rs b/crates/sail-plan/src/function/metadata.rs
new file mode 100644
index 000000000..ac07d575a
--- /dev/null
+++ b/crates/sail-plan/src/function/metadata.rs
@@ -0,0 +1,58 @@
+use std::collections::HashMap;
+
+use lazy_static::lazy_static;
+use sail_common_datafusion::catalog::FunctionStatus;
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct BuiltInFunctionMetadata {
+    pub name: &'static str,
+    pub signatures: &'static [&'static str],
+    pub usage: Option<&'static str>,
+    pub arguments: Option<&'static str>,
+    pub examples: Option<&'static str>,
+    pub note: Option<&'static str>,
+    pub since: Option<&'static str>,
+    pub deprecated: Option<&'static str>,
+    pub class_name: &'static str,
+    pub is_public: bool,
+}
+
+include!(concat!(env!("OUT_DIR"), "/function_metadata.rs"));
+
+lazy_static! {
+    // TODO: Generate this lookup as a `phf::Map`.
+    static ref BUILT_IN_FUNCTION_METADATA_BY_NAME: HashMap<&'static str, BuiltInFunctionMetadata> =
+        HashMap::from_iter(
+            BUILT_IN_FUNCTION_METADATA
+                .iter()
+                .map(|metadata| (metadata.name, *metadata))
+        );
+}
+
+fn built_in_function_status(metadata: &BuiltInFunctionMetadata) -> FunctionStatus {
+    FunctionStatus {
+        name: metadata.name.to_string(),
+        catalog: None,
+        namespace: None,
+        signatures: metadata
+            .signatures
+            .iter()
+            .map(|signature| signature.to_string())
+            .collect(),
+        usage: metadata.usage.map(|usage| usage.to_string()),
+        arguments: metadata.arguments.map(|arguments| arguments.to_string()),
+        examples: metadata.examples.map(|examples| examples.to_string()),
+        note: metadata.note.map(|note| note.to_string()),
+        since: metadata.since.map(|since| since.to_string()),
+        deprecated: metadata.deprecated.map(|deprecated| deprecated.to_string()),
+        class_name: metadata.class_name.to_string(),
+        is_temporary: true,
+    }
+}
+
+pub(crate) fn built_in_public_function_status(name: &str) -> Option {
+    let metadata = BUILT_IN_FUNCTION_METADATA_BY_NAME.get(name)?;
+    metadata
+        .is_public
+        .then(|| built_in_function_status(metadata))
+}
diff --git a/crates/sail-plan/src/function/mod.rs b/crates/sail-plan/src/function/mod.rs
index 82dad8087..38a03ae7c 100644
--- a/crates/sail-plan/src/function/mod.rs
+++ b/crates/sail-plan/src/function/mod.rs
@@ -5,6 +5,7 @@ use datafusion::catalog::TableFunction;
 use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
 use datafusion_expr::expr::Expr;
 use lazy_static::lazy_static;
+use sail_common_datafusion::catalog::FunctionStatus;
 
 use crate::error::{PlanError, PlanResult};
 use crate::function::common::ScalarFunction;
@@ -12,6 +13,7 @@ use crate::function::common::ScalarFunction;
 mod aggregate;
 pub(crate) mod common;
 mod generator;
+mod metadata;
 mod scalar;
 mod table;
 mod window;
@@ -29,6 +31,8 @@ lazy_static! {
         HashMap::from_iter(table::list_built_in_table_functions());
 }
 
+const BUILT_IN_OPERATOR_FUNCTION_NAMES: &[&str] = &["<>", "between", "||"];
+
 pub fn get_built_in_function(name: &str) -> PlanResult {
     Ok(BUILT_IN_SCALAR_FUNCTIONS
         .get(name)
@@ -48,6 +52,28 @@ pub fn is_built_in_generator_function(name: &str) -> bool {
     BUILT_IN_GENERATOR_FUNCTIONS.contains_key(name)
 }
 
+fn list_built_in_function_names() -> Vec<&'static str> {
+    let mut names = BUILT_IN_SCALAR_FUNCTIONS
+        .keys()
+        .chain(BUILT_IN_GENERATOR_FUNCTIONS.keys())
+        .chain(BUILT_IN_TABLE_FUNCTIONS.keys())
+        .copied()
+        .chain(aggregate::list_built_in_aggregate_function_names())
+        .chain(window::list_built_in_window_function_names())
+        .chain(BUILT_IN_OPERATOR_FUNCTION_NAMES.iter().copied())
+        .collect::>();
+    names.sort_unstable();
+    names.dedup();
+    names
+}
+
+pub(crate) fn list_built_in_function_statuses() -> Vec {
+    list_built_in_function_names()
+        .into_iter()
+        .filter_map(metadata::built_in_public_function_status)
+        .collect()
+}
+
 pub use generator::get_outer_built_in_generator_functions;
 
 /// This function is temporary and should ONLY be used for COUNT(*).
@@ -61,10 +87,12 @@ pub use generator::get_outer_built_in_generator_functions;
 pub(super) fn transform_count_star_wildcard_expr(arguments: Vec) -> Vec {
     match arguments.as_slice() {
         #[expect(deprecated)]
-        [Expr::Wildcard {
-            qualifier: None,
-            options: _,
-        }] => {
+        [
+            Expr::Wildcard {
+                qualifier: None,
+                options: _,
+            },
+        ] => {
             vec![Expr::Literal(COUNT_STAR_EXPANSION, None)]
         }
         _ => arguments,
diff --git a/crates/sail-plan/src/function/scalar/array.rs b/crates/sail-plan/src/function/scalar/array.rs
index f80143019..8766cc32d 100644
--- a/crates/sail-plan/src/function/scalar/array.rs
+++ b/crates/sail-plan/src/function/scalar/array.rs
@@ -4,7 +4,7 @@ use datafusion::arrow::datatypes::DataType;
 use datafusion::functions::expr_fn::{coalesce, nvl};
 use datafusion::functions_nested::expr_fn;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{cast, expr, is_null, lit, not, or, when, ExprSchemable, ScalarUDF};
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, is_null, lit, not, or, when};
 use datafusion_functions_nested::make_array::make_array;
 use datafusion_functions_nested::string::ArrayToString;
 use datafusion_spark::function::array::expr_fn as array_fn;
@@ -100,7 +100,7 @@ fn sort_array(input: ScalarFunctionInput) -> PlanResult {
         n => {
             return Err(PlanError::invalid(format!(
                 "sort_array requires 1 or 2 arguments, got {n}"
-            )))
+            )));
         }
     };
     let (sort, nulls) = match asc {
@@ -115,7 +115,7 @@ fn sort_array(input: ScalarFunctionInput) -> PlanResult {
         _ => {
             return Err(PlanError::invalid(format!(
                 "Invalid asc value for sort_array: {asc}"
-            )))
+            )));
         }
     };
     Ok(expr_fn::array_sort(array, sort, nulls))
@@ -415,7 +415,7 @@ mod tests {
     use datafusion::arrow::datatypes::Int32Type;
     use datafusion::prelude::SessionContext;
     use datafusion_common::DFSchema;
-    use datafusion_expr::{col, ColumnarValue};
+    use datafusion_expr::{ColumnarValue, col};
 
     use super::*;
 
diff --git a/crates/sail-plan/src/function/scalar/bitwise.rs b/crates/sail-plan/src/function/scalar/bitwise.rs
index 6a8a79457..ba38291ab 100644
--- a/crates/sail-plan/src/function/scalar/bitwise.rs
+++ b/crates/sail-plan/src/function/scalar/bitwise.rs
@@ -1,6 +1,6 @@
 use datafusion::arrow::datatypes::DataType;
 use datafusion::functions::expr_fn::abs;
-use datafusion_expr::{cast, expr, lit, when, ExprSchemable, Operator};
+use datafusion_expr::{ExprSchemable, Operator, cast, expr, lit, when};
 use datafusion_spark::function::bitwise::expr_fn as bitwise_fn;
 use sail_common_datafusion::utils::items::ItemTaker;
 
diff --git a/crates/sail-plan/src/function/scalar/collection.rs b/crates/sail-plan/src/function/scalar/collection.rs
index 970f91a52..2e72aff47 100644
--- a/crates/sail-plan/src/function/scalar/collection.rs
+++ b/crates/sail-plan/src/function/scalar/collection.rs
@@ -1,6 +1,6 @@
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{cast, expr, lit, when, ExprSchemable, ScalarUDF};
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, lit, when};
 use datafusion_functions::math::expr_fn::abs;
 use datafusion_functions_nested::expr_fn;
 use sail_common_datafusion::utils::items::ItemTaker;
diff --git a/crates/sail-plan/src/function/scalar/conditional.rs b/crates/sail-plan/src/function/scalar/conditional.rs
index 7fc6550c4..2d0bbfe70 100644
--- a/crates/sail-plan/src/function/scalar/conditional.rs
+++ b/crates/sail-plan/src/function/scalar/conditional.rs
@@ -1,7 +1,7 @@
 use arrow::datatypes::{DataType, TimeUnit};
 use datafusion::functions::expr_fn;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{cast, expr, lit, ExprSchemable, ScalarUDF};
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, lit};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::datetime::spark_date::SparkDate;
 use sail_function::scalar::datetime::spark_timestamp::SparkTimestamp;
@@ -15,11 +15,14 @@ fn case(input: ScalarFunctionInput) -> PlanResult {
     let mut when_then_expr = Vec::new();
     let mut iter = arguments.into_iter();
     while let Some(condition) = iter.next() {
-        if let Some(result) = iter.next() {
-            when_then_expr.push((Box::new(condition), Box::new(result)));
-        } else {
-            when_then_expr.push((Box::new(lit(true)), Box::new(condition)));
-            break;
+        match iter.next() {
+            Some(result) => {
+                when_then_expr.push((Box::new(condition), Box::new(result)));
+            }
+            _ => {
+                when_then_expr.push((Box::new(lit(true)), Box::new(condition)));
+                break;
+            }
         }
     }
     Ok(expr::Expr::Case(expr::Case {
@@ -91,11 +94,13 @@ fn coerce_to_temporal(
     if is_string_type(data_type) {
         match target_type {
             DataType::Date32 => Ok(ScalarUDF::from(SparkDate::new(false)).call(vec![arg])),
-            DataType::Timestamp(_, timezone) => Ok(ScalarUDF::from(SparkTimestamp::try_new(
-                timezone.clone(),
-                false,
-            )?)
-            .call(vec![arg])),
+            // Only reached on the ANSI-enabled coalesce path, so strict parsing.
+            DataType::Timestamp(_, timezone) => {
+                Ok(
+                    ScalarUDF::from(SparkTimestamp::try_new(timezone.clone(), true, false)?)
+                        .call(vec![arg]),
+                )
+            }
             _ => Ok(cast(arg, target_type.clone())),
         }
     } else if is_temporal_type(data_type) {
diff --git a/crates/sail-plan/src/function/scalar/conversion.rs b/crates/sail-plan/src/function/scalar/conversion.rs
index c612725e7..955c77ec6 100644
--- a/crates/sail-plan/src/function/scalar/conversion.rs
+++ b/crates/sail-plan/src/function/scalar/conversion.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use datafusion::arrow::datatypes::{DataType, TimeUnit};
-use datafusion_expr::{expr, ExprSchemable, ScalarUDF};
+use datafusion_expr::{ExprSchemable, ScalarUDF, expr};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::datetime::spark_date::SparkDate;
 use sail_function::scalar::datetime::spark_time::SparkTime;
@@ -70,6 +70,7 @@ fn cast_to_timestamp(input: ScalarFunctionInput) -> PlanResult {
         Ok(expr::Expr::ScalarFunction(expr::ScalarFunction {
             func: Arc::new(ScalarUDF::from(SparkTimestamp::try_new(
                 Some(input.function_context.plan_config.session_timezone.clone()),
+                input.function_context.plan_config.ansi_mode,
                 false,
             )?)),
             args: vec![arg],
diff --git a/crates/sail-plan/src/function/scalar/csv.rs b/crates/sail-plan/src/function/scalar/csv.rs
index 8f49a9291..bf176a3b5 100644
--- a/crates/sail-plan/src/function/scalar/csv.rs
+++ b/crates/sail-plan/src/function/scalar/csv.rs
@@ -1,8 +1,8 @@
-use datafusion_expr::{expr, Expr, ScalarUDF};
+use datafusion_expr::{Expr, ScalarUDF, expr};
 use sail_common_datafusion::literal::LiteralEvaluator;
+use sail_function::scalar::csv::SparkSchemaOfCsv;
 use sail_function::scalar::csv::spark_from_csv::SparkFromCSV;
 use sail_function::scalar::csv::spark_to_csv::SparkToCsv;
-use sail_function::scalar::csv::SparkSchemaOfCsv;
 
 use crate::error::PlanResult;
 use crate::function::common::{ScalarFunction, ScalarFunctionBuilder as F, ScalarFunctionInput};
diff --git a/crates/sail-plan/src/function/scalar/datetime.rs b/crates/sail-plan/src/function/scalar/datetime.rs
index 27302c049..9ad8a63d6 100644
--- a/crates/sail-plan/src/function/scalar/datetime.rs
+++ b/crates/sail-plan/src/function/scalar/datetime.rs
@@ -6,14 +6,15 @@ use datafusion::arrow::datatypes::{
 use datafusion::functions::expr_fn;
 use datafusion_common::{DFSchemaRef, ScalarValue};
 use datafusion_expr::expr::{self, Expr};
-use datafusion_expr::{cast, lit, try_cast, when, BinaryExpr, ExprSchemable, Operator, ScalarUDF};
+use datafusion_expr::{BinaryExpr, ExprSchemable, Operator, ScalarUDF, cast, lit, try_cast, when};
 use datafusion_functions::core::expr_ext::FieldAccessor;
-use datafusion_functions::expr_fn::to_time;
 use datafusion_spark::function::datetime::make_dt_interval::SparkMakeDtInterval;
 use datafusion_spark::function::datetime::make_interval::SparkMakeInterval;
-use sail_common::datetime::time_unit_to_multiplier;
+use sail_common::utils::datetime::time_unit_to_multiplier;
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::datetime::convert_tz::ConvertTz;
+use sail_function::scalar::datetime::spark_date::SparkDate;
+use sail_function::scalar::datetime::spark_date_format::SparkDateFormat;
 use sail_function::scalar::datetime::spark_date_part::SparkDatePart;
 use sail_function::scalar::datetime::spark_date_trunc::SparkDateTrunc;
 use sail_function::scalar::datetime::spark_last_day::SparkLastDay;
@@ -21,11 +22,10 @@ use sail_function::scalar::datetime::spark_make_time::SparkMakeTime;
 use sail_function::scalar::datetime::spark_make_timestamp_ntz::SparkMakeTimestampNtz;
 use sail_function::scalar::datetime::spark_make_ym_interval::SparkMakeYmInterval;
 use sail_function::scalar::datetime::spark_next_day::SparkNextDay;
+use sail_function::scalar::datetime::spark_time::SparkTime;
 use sail_function::scalar::datetime::spark_time_diff::SparkTimeDiff;
 use sail_function::scalar::datetime::spark_time_trunc::SparkTimeTrunc;
 use sail_function::scalar::datetime::spark_timestamp::SparkTimestamp;
-use sail_function::scalar::datetime::spark_to_chrono_fmt::SparkToChronoFmt;
-use sail_function::scalar::datetime::spark_try_to_timestamp::SparkTryToTimestamp;
 use sail_function::scalar::datetime::spark_unix_timestamp::SparkUnixTimestamp;
 use sail_function::scalar::datetime::spark_window_buckets::SparkWindowBuckets;
 use sail_function::scalar::datetime::spark_year::SparkYear;
@@ -138,7 +138,7 @@ fn interval_arithmetic(input: ScalarFunctionInput, unit: &str, op: Operator) ->
         _ => {
             return Err(PlanError::invalid(format!(
                 "add_interval does not support interval unit type '{unit}'"
-            )))
+            )));
         }
     };
     Ok(Expr::BinaryExpr(BinaryExpr {
@@ -229,7 +229,7 @@ fn timestampadd(input: ScalarFunctionInput) -> PlanResult {
         _ => {
             return Err(PlanError::invalid(
                 "timestampadd unit must be a string literal or keyword",
-            ))
+            ));
         }
     };
     let interval = timestampadd_interval(&unit, quantity)?;
@@ -269,6 +269,83 @@ fn date_days_arithmetic(dt1: Expr, dt2: Expr, op: Operator) -> Expr {
     })
 }
 
+fn timestamp_micros(expr: Expr) -> Expr {
+    cast(
+        cast(expr, DataType::Timestamp(TimeUnit::Microsecond, None)),
+        DataType::Int64,
+    )
+}
+
+fn timestamp_time_micros(expr: Expr) -> Expr {
+    timestamp_micros(expr.clone()) - timestamp_micros(expr_fn::date_trunc(lit("DAY"), expr))
+}
+
+fn timestamp_months(expr: Expr) -> Expr {
+    integer_part(expr.clone(), "YEAR") * lit(12) + integer_part(expr, "MONTH")
+}
+
+fn timestamp_day_time_is_before(left: Expr, right: Expr) -> Expr {
+    let left_day = integer_part(left.clone(), "DAY");
+    let right_day = integer_part(right.clone(), "DAY");
+    left_day.clone().lt(right_day.clone()).or(left_day
+        .eq(right_day)
+        .and(timestamp_time_micros(left).lt(timestamp_time_micros(right))))
+}
+
+fn timestamp_month_diff(start: Expr, end: Expr) -> PlanResult {
+    let start = cast(start, DataType::Timestamp(TimeUnit::Microsecond, None));
+    let end = cast(end, DataType::Timestamp(TimeUnit::Microsecond, None));
+    let months = cast(
+        timestamp_months(end.clone()) - timestamp_months(start.clone()),
+        DataType::Int64,
+    );
+    let incomplete_positive_month = months
+        .clone()
+        .gt(lit(0_i64))
+        .and(timestamp_day_time_is_before(end.clone(), start.clone()));
+    let incomplete_negative_month = months
+        .clone()
+        .lt(lit(0_i64))
+        .and(timestamp_day_time_is_before(start, end));
+    Ok(when(incomplete_positive_month, months.clone() - lit(1_i64))
+        .when(incomplete_negative_month, months.clone() + lit(1_i64))
+        .when(lit(true), months)
+        .end()?)
+}
+
+fn timestampdiff_calendar_unit(unit: &str, start: Expr, end: Expr) -> PlanResult {
+    let months = timestamp_month_diff(start, end)?;
+    match unit {
+        "MONTH" => Ok(months),
+        "QUARTER" => Ok(months / lit(3_i64)),
+        "YEAR" => Ok(months / lit(12_i64)),
+        _ => Err(PlanError::internal(format!(
+            "invalid timestampdiff calendar unit: {unit}"
+        ))),
+    }
+}
+
+fn timestampdiff_fixed_unit(unit: &str, start: Expr, end: Expr) -> Expr {
+    let start_ts = timestamp_micros(start);
+    let end_ts = timestamp_micros(end);
+    let diff_micros = cast(
+        Expr::BinaryExpr(BinaryExpr {
+            left: Box::new(end_ts),
+            op: Operator::Minus,
+            right: Box::new(start_ts),
+        }),
+        DataType::Int64,
+    );
+    let divisor = match unit {
+        "SECOND" => 1_000_000i64,
+        "MINUTE" => 60_000_000i64,
+        "HOUR" => 3_600_000_000i64,
+        "WEEK" => 7 * 24 * 3_600_000_000i64,
+        _ => 1i64,
+    };
+    diff_micros / lit(divisor)
+}
+
 fn datediff(input: ScalarFunctionInput) -> PlanResult {
     let args = input.arguments;
     match args.len() {
@@ -287,38 +364,15 @@ fn datediff(input: ScalarFunctionInput) -> PlanResult {
                 _ => {
                     return Err(PlanError::invalid(
                         "datediff unit must be a string literal or keyword",
-                    ))
+                    ));
                 }
             };
             match unit_str.as_str() {
                 "DAY" => Ok(date_days_arithmetic(end, start, Operator::Minus)),
-                "HOUR" | "MINUTE" | "SECOND" | "MONTH" | "YEAR" | "WEEK" | "QUARTER" => {
-                    let start_ts = cast(start, DataType::Timestamp(TimeUnit::Microsecond, None));
-                    let end_ts = cast(end, DataType::Timestamp(TimeUnit::Microsecond, None));
-                    let diff_seconds = cast(
-                        Expr::BinaryExpr(BinaryExpr {
-                            left: Box::new(cast(end_ts, DataType::Int64)),
-                            op: Operator::Minus,
-                            right: Box::new(cast(start_ts, DataType::Int64)),
-                        }),
-                        DataType::Int64,
-                    );
-                    let divisor = match unit_str.as_str() {
-                        "SECOND" => 1_000_000i64,
-                        "MINUTE" => 60_000_000i64,
-                        "HOUR" => 3_600_000_000i64,
-                        "WEEK" => 7 * 24 * 3_600_000_000i64,
-                        "MONTH" => 30 * 24 * 3_600_000_000i64,
-                        "YEAR" => 365 * 24 * 3_600_000_000i64,
-                        "QUARTER" => 91 * 24 * 3_600_000_000i64,
-                        _ => 1i64,
-                    };
-                    Ok(Expr::BinaryExpr(BinaryExpr {
-                        left: Box::new(diff_seconds),
-                        op: Operator::Divide,
-                        right: Box::new(lit(divisor)),
-                    }))
+                "HOUR" | "MINUTE" | "SECOND" | "WEEK" => {
+                    Ok(timestampdiff_fixed_unit(&unit_str, start, end))
                 }
+                "MONTH" | "YEAR" | "QUARTER" => timestampdiff_calendar_unit(&unit_str, start, end),
                 other => Err(PlanError::unsupported(format!("datediff unit: {other}"))),
             }
         }
@@ -342,8 +396,30 @@ fn current_timezone(input: ScalarFunctionInput) -> PlanResult {
     Ok(session_tz)
 }
 
-fn to_chrono_fmt(format: Expr) -> Expr {
-    ScalarUDF::from(SparkToChronoFmt::new()).call(vec![format])
+fn coerce_datetime_format(
+    function_name: &str,
+    format: Expr,
+    schema: &DFSchemaRef,
+) -> PlanResult {
+    let data_type = format.get_type(schema)?;
+    if data_type.is_nested() {
+        Err(PlanError::invalid(format!(
+            "{function_name} format argument must be a string, got {data_type}"
+        )))
+    } else if data_type.is_string() {
+        Ok(format)
+    } else {
+        Ok(cast(format, DataType::Utf8))
+    }
+}
+
+fn declare_nullable_result(expr: Expr, nullable: bool, schema: &DFSchemaRef) -> PlanResult {
+    if nullable && !expr.nullable(schema)? {
+        let null = lit(ScalarValue::try_from(&expr.get_type(schema)?)?);
+        Ok(when(lit(true), expr).otherwise(null)?)
+    } else {
+        Ok(expr)
+    }
 }
 
 fn to_date(input: ScalarFunctionInput) -> PlanResult {
@@ -351,43 +427,83 @@ fn to_date(input: ScalarFunctionInput) -> PlanResult {
         // If format is not supplied, the function is a synonym for cast(expr AS DATE).
         crate::function::scalar::conversion::cast_to_date(input)
     } else if input.arguments.len() == 2 {
-        let (expr, format) = input.arguments.two()?;
+        let expr = input.arguments[0].clone();
+        let format = coerce_datetime_format(
+            "to_date",
+            input.arguments[1].clone(),
+            input.function_context.schema,
+        )?;
         let expr_type = expr.get_type(input.function_context.schema);
-        if let Ok(DataType::Timestamp(_, _)) = expr_type {
-            let expr = expr_fn::to_local_time(vec![expr]);
-            return Ok(cast(expr, DataType::Date32)); // In case of data type timestamp, ignore format
+        let date = match &expr_type {
+            Ok(DataType::Timestamp(_, _)) => Some(expr_fn::to_local_time(vec![expr.clone()])),
+            Ok(DataType::Date32 | DataType::Date64) => Some(expr.clone()),
+            _ => None,
+        };
+        if let Some(date) = date {
+            let nullable = !input.function_context.plan_config.ansi_mode
+                || expr.nullable(input.function_context.schema)?
+                || format.nullable(input.function_context.schema)?;
+            return declare_nullable_result(
+                cast(date, DataType::Date32),
+                nullable,
+                input.function_context.schema,
+            );
         }
         let expr = match expr_type {
             Ok(_other) => expr,
             Err(_) => cast(expr, DataType::Utf8), // In case of error, cast to string
         };
-        let format = to_chrono_fmt(format);
-        Ok(expr_fn::to_date(vec![expr, format]))
+        Ok(ScalarUDF::from(SparkDate::new(false)).call(vec![expr, format]))
     } else {
         Err(PlanError::invalid("to_date requires 1 or 2 arguments"))
     }
 }
 
+fn validate_unix_timestamp_format(format: &Expr, schema: &DFSchemaRef) -> PlanResult<()> {
+    match format.get_type(schema)? {
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View | DataType::Null => Ok(()),
+        data_type => Err(PlanError::invalid(format!(
+            "unix_timestamp format argument must be a string, got {data_type}"
+        ))),
+    }
+}
+
 fn unix_timestamp(input: ScalarFunctionInput) -> PlanResult {
     let timezone = input.function_context.plan_config.session_timezone.clone();
+    let ansi_mode = input.function_context.plan_config.ansi_mode;
     if input.arguments.is_empty() {
         let expr = ScalarUDF::from(TimestampNow::new(timezone, TimeUnit::Second)).call(vec![]);
         Ok(cast(expr, DataType::Int64))
     } else if input.arguments.len() == 1 {
-        Ok(ScalarUDF::from(SparkUnixTimestamp::new(timezone)).call(input.arguments))
+        Ok(ScalarUDF::from(SparkUnixTimestamp::new(timezone, ansi_mode)).call(input.arguments))
     } else if input.arguments.len() == 2 {
         let (expr, format) = input.arguments.two()?;
-        let format = to_chrono_fmt(format);
-        Ok(ScalarUDF::from(SparkUnixTimestamp::new(timezone)).call(vec![expr, format]))
+        if matches!(
+            expr.get_type(input.function_context.schema)?,
+            DataType::Date32 | DataType::Date64 | DataType::Timestamp(_, _) | DataType::Null
+        ) {
+            validate_unix_timestamp_format(&format, input.function_context.schema)?;
+            let nullable = !ansi_mode
+                || expr.nullable(input.function_context.schema)?
+                || format.nullable(input.function_context.schema)?;
+            let result =
+                ScalarUDF::from(SparkUnixTimestamp::new(timezone, ansi_mode)).call(vec![expr]);
+            declare_nullable_result(result, nullable, input.function_context.schema)
+        } else {
+            Ok(
+                ScalarUDF::from(SparkUnixTimestamp::new(timezone, ansi_mode))
+                    .call(vec![expr, format]),
+            )
+        }
     } else {
         Err(PlanError::invalid(
-            "unix_timestamp requires 1 or 2 arguments",
+            "unix_timestamp requires 0, 1, or 2 arguments",
         ))
     }
 }
 
 fn to_unix_timestamp(input: ScalarFunctionInput) -> PlanResult {
-    if input.arguments.is_empty() {
+    if !(1..=2).contains(&input.arguments.len()) {
         Err(PlanError::invalid(
             "to_unix_timestamp requires 1 or 2 arguments",
         ))
@@ -407,22 +523,12 @@ fn next_day(input: ScalarFunctionInput) -> PlanResult {
     Ok(udf.call(input.arguments))
 }
 
-pub(super) fn date_format(expr: Expr, format: Expr) -> Expr {
-    // Handle standalone fractional seconds format (e.g., 'SSS' for milliseconds).
-    // Chrono's %.Nf always includes a leading dot (e.g., ".000"), so for standalone
-    // S-patterns we strip the dot using substr.
-    if let Expr::Literal(ref sv, _) = &format {
-        if let Some(Some(fmt)) = sv.try_as_str() {
-            if !fmt.is_empty() && fmt.chars().all(|c| c == 'S') {
-                let n = fmt.len();
-                let chrono_fmt = format!("%.{n}f");
-                let result = expr_fn::to_char(expr, lit(chrono_fmt));
-                return expr_fn::substr(result, lit(2i64));
-            }
-        }
-    }
-    let format = to_chrono_fmt(format);
-    expr_fn::to_char(expr, format)
+pub(super) fn date_format(expr: Expr, format: Expr, timezone: String) -> Expr {
+    date_format_with_args(vec![expr, format], timezone)
+}
+
+fn date_format_with_args(arguments: Vec, timezone: String) -> Expr {
+    ScalarUDF::from(SparkDateFormat::new(timezone.into())).call(arguments)
 }
 
 fn timestamp_data_type(input: &ScalarFunctionInput, timestamp_ntz: bool) -> DataType {
@@ -448,54 +554,114 @@ fn is_null_literal(expr: &Expr) -> bool {
 }
 
 fn to_timestamp(input: ScalarFunctionInput, timestamp_ntz: bool) -> PlanResult {
-    let data_type = timestamp_data_type(&input, timestamp_ntz);
+    timestamp_with_try(input, timestamp_ntz, false)
+}
+
+fn to_time(input: ScalarFunctionInput) -> PlanResult {
+    time_with_try(input, false)
+}
+
+fn try_to_time(input: ScalarFunctionInput) -> PlanResult {
+    time_with_try(input, true)
+}
+
+/// Shared `to_time` / `try_to_time` planner. Routes through `SparkTime`, which
+/// parses strings (with an optional Spark Java datetime pattern) or casts
+/// time/timestamp args.
+/// `to_time` errors on failure (Spark's `ToTime` is ANSI-invariant); `try_to_time`
+/// (`is_try`) returns NULL.
+fn time_with_try(input: ScalarFunctionInput, is_try: bool) -> PlanResult {
+    let udf = ScalarUDF::from(SparkTime::new(is_try));
     if input.arguments.len() == 1 {
-        let expr = input.arguments.one()?;
-        let expr = match expr.get_type(input.function_context.schema)? {
-            DataType::Timestamp(_, Some(_)) => expr_fn::to_local_time(vec![expr]),
-            _ => expr,
-        };
-        Ok(cast(expr, data_type))
+        Ok(udf.call(input.arguments))
     } else if input.arguments.len() == 2 {
-        let null = timestamp_null(&input, timestamp_ntz);
+        // Pass `expr` through unchanged so `SparkTime::coerce_types` validates it
+        // and the kernel dispatches by type (strings parse with the format,
+        // TIME/TIMESTAMP cast directly), exactly as in the 1-arg form. Forcing a
+        // cast to Utf8 here would route non-string inputs through string parsing,
+        // bypassing the coercion checks and diverging from the 1-arg behavior.
         let (expr, format) = input.arguments.two()?;
-        if is_null_literal(&expr) || is_null_literal(&format) {
-            return Ok(null);
-        }
-        let format = to_chrono_fmt(format);
-        Ok(cast(
-            expr_fn::to_timestamp_micros(vec![expr, format]),
-            data_type,
-        ))
+        Ok(udf.call(vec![expr, format]))
     } else {
-        Err(PlanError::invalid("to_timestamp requires 1 or 2 arguments"))
+        let name = if is_try { "try_to_time" } else { "to_time" };
+        Err(PlanError::invalid(format!(
+            "{name} requires 1 or 2 arguments"
+        )))
     }
 }
 
 fn try_to_timestamp(input: ScalarFunctionInput, timestamp_ntz: bool) -> PlanResult {
+    timestamp_with_try(input, timestamp_ntz, true)
+}
+
+/// Shared `to_timestamp` / `try_to_timestamp` (+ `_ntz`) planner.
+///
+/// The 1-arg form goes through `cast` / `try_cast`, which route strings to
+/// `SparkTimestamp` (honoring ANSI for the strict variant) and cast other types.
+/// The 2-arg form parses the value with the given format via `SparkTimestamp`.
+fn timestamp_with_try(
+    input: ScalarFunctionInput,
+    timestamp_ntz: bool,
+    is_try: bool,
+) -> PlanResult {
     let data_type = timestamp_data_type(&input, timestamp_ntz);
+    let ansi_mode = input.function_context.plan_config.ansi_mode;
+    let timezone = if timestamp_ntz {
+        None
+    } else {
+        Some(input.function_context.plan_config.session_timezone.clone())
+    };
     if input.arguments.len() == 1 {
         let expr = input.arguments.one()?;
-        let expr = match expr.get_type(input.function_context.schema)? {
-            DataType::Timestamp(_, Some(_)) => expr_fn::to_local_time(vec![expr]),
-            _ => expr,
-        };
-        Ok(try_cast(expr, data_type))
+        let expr_type = expr.get_type(input.function_context.schema)?;
+        if matches!(
+            expr_type,
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+        ) {
+            // Strings parse through SparkTimestamp, which honors ANSI (errors
+            // under ANSI, NULL otherwise) for the strict variant.
+            let udf = ScalarUDF::from(SparkTimestamp::try_new(timezone, ansi_mode, is_try)?);
+            Ok(udf.call(vec![expr]))
+        } else {
+            // Timestamp-with-tz is re-based to the session zone; other types cast.
+            let expr = match expr_type {
+                DataType::Timestamp(_, Some(_)) => expr_fn::to_local_time(vec![expr]),
+                _ => expr,
+            };
+            if is_try {
+                Ok(try_cast(expr, data_type))
+            } else {
+                Ok(cast(expr, data_type))
+            }
+        }
     } else if input.arguments.len() == 2 {
         let null = timestamp_null(&input, timestamp_ntz);
-        let (expr, format) = input.arguments.two()?;
-        if is_null_literal(&expr) || is_null_literal(&format) {
+        if input.arguments.iter().any(is_null_literal) {
             return Ok(null);
         }
-        let format = to_chrono_fmt(format);
+        let arguments = input.arguments;
         Ok(cast(
-            ScalarUDF::from(SparkTryToTimestamp::new()).call(vec![expr, format]),
+            ScalarUDF::from(SparkTimestamp::try_new(
+                match &data_type {
+                    DataType::Timestamp(_, timezone) => timezone.clone(),
+                    _ => None,
+                },
+                ansi_mode,
+                is_try,
+            )?)
+            .call(arguments),
             data_type,
         ))
     } else {
-        Err(PlanError::invalid(
-            "try_to_timestamp requires 1 or 2 arguments",
-        ))
+        let name = match (is_try, timestamp_ntz) {
+            (false, false) => "to_timestamp",
+            (true, false) => "try_to_timestamp",
+            (false, true) => "to_timestamp_ntz",
+            (true, true) => "try_to_timestamp_ntz",
+        };
+        Err(PlanError::invalid(format!(
+            "{name} requires 1 or 2 arguments"
+        )))
     }
 }
 
@@ -513,9 +679,11 @@ fn from_unixtime(input: ScalarFunctionInput) -> PlanResult {
     }?;
 
     let timezone = input.function_context.plan_config.session_timezone.clone();
-    let format = to_chrono_fmt(format);
-    let expr = cast(expr, DataType::Timestamp(TimeUnit::Second, Some(timezone)));
-    Ok(expr_fn::to_char(expr, format))
+    let expr = cast(
+        expr,
+        DataType::Timestamp(TimeUnit::Second, Some(timezone.clone())),
+    );
+    Ok(date_format(expr, format, timezone.to_string()))
 }
 
 fn unix_time_unit(input: ScalarFunctionInput, time_unit: TimeUnit) -> PlanResult {
@@ -568,8 +736,8 @@ fn ntz_timestamp_and_unit(
         }
         DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
             let unit = TimeUnit::Microsecond;
-            let is_try = !ansi_mode;
-            let ts = ScalarUDF::from(SparkTimestamp::try_new(None, is_try)?).call(vec![ts]);
+            let ts =
+                ScalarUDF::from(SparkTimestamp::try_new(None, ansi_mode, false)?).call(vec![ts]);
             Ok((ts, unit))
         }
         x => Err(PlanError::invalid(format!(
@@ -626,7 +794,7 @@ fn utc_ntz_timestamp_and_unit(
         x => {
             return Err(PlanError::invalid(format!(
                 "invalid UTC NTZ timestamp type: {x:?}"
-            )))
+            )));
         }
     };
     let ts = cast(ts, DataType::Timestamp(unit, None));
@@ -651,15 +819,6 @@ fn to_utc_timestamp(input: ScalarFunctionInput) -> PlanResult {
     Ok(cast(ts, DataType::Timestamp(unit, Some(session_tz))))
 }
 
-fn make_ym_interval(args: Vec) -> PlanResult {
-    let (years, months) = if args.len() == 2 {
-        args.two()?
-    } else {
-        (args.one()?, lit(0_i32))
-    };
-    Ok(ScalarUDF::from(SparkMakeYmInterval::new()).call(vec![years, months]))
-}
-
 fn make_timestamp_ltz(args: Vec, session_tz: &Arc, is_try: bool) -> PlanResult {
     let ntz_ts = if args.len() == 2 || args.len() == 6 {
         ScalarUDF::from(SparkMakeTimestampNtz::new(is_try)).call(args)
@@ -914,7 +1073,7 @@ fn window_field_type(
         other => {
             return Err(PlanError::invalid(format!(
                 "window requires a timestamp time column, got {other:?}"
-            )))
+            )));
         }
     })
 }
@@ -967,7 +1126,7 @@ fn window_time(input: ScalarFunctionInput) -> PlanResult {
         other => {
             return Err(PlanError::invalid(format!(
                 "window_time requires a window column (struct with start and end timestamps), got {other:?}"
-            )))
+            )));
         }
     };
     Ok(cast(
@@ -1006,7 +1165,16 @@ pub(super) fn list_built_in_datetime_functions() -> Vec<(&'static str, ScalarFun
             F::custom(|input| interval_arithmetic(input, "days", Operator::Plus)),
         ),
         ("date_diff", F::custom(datediff)),
-        ("date_format", F::binary(date_format)),
+        (
+            "date_format",
+            F::custom(|input| match input.arguments.len() {
+                2 => {
+                    let timezone = input.function_context.plan_config.session_timezone.clone();
+                    Ok(date_format_with_args(input.arguments, timezone.to_string()))
+                }
+                _ => Err(PlanError::invalid("date_format requires 2 arguments")),
+            }),
+        ),
         ("date_from_unix_date", F::cast(DataType::Date32)),
         ("date_part", F::binary(date_part)),
         (
@@ -1059,7 +1227,7 @@ pub(super) fn list_built_in_datetime_functions() -> Vec<(&'static str, ScalarFun
             "make_timestamp_ntz",
             F::custom(|input| make_timestamp_ntz(input.arguments, false)),
         ),
-        ("make_ym_interval", F::var_arg(make_ym_interval)),
+        ("make_ym_interval", F::udf(SparkMakeYmInterval::new())),
         ("minute", F::unary(|arg| integer_part(arg, "MINUTE"))),
         ("month", F::unary(|arg| integer_part(arg, "MONTH"))),
         (
@@ -1072,6 +1240,13 @@ pub(super) fn list_built_in_datetime_functions() -> Vec<(&'static str, ScalarFun
         ("quarter", F::unary(|arg| integer_part(arg, "QUARTER"))),
         ("second", F::unary(|arg| integer_part(arg, "SECOND"))),
         ("session_window", F::unknown("session_window")),
+        ("time_bucket", F::unknown("time_bucket")),
+        ("time_from_micros", F::unknown("time_from_micros")),
+        ("time_from_millis", F::unknown("time_from_millis")),
+        ("time_from_seconds", F::unknown("time_from_seconds")),
+        ("time_to_micros", F::unknown("time_to_micros")),
+        ("time_to_millis", F::unknown("time_to_millis")),
+        ("time_to_seconds", F::unknown("time_to_seconds")),
         (
             "timestamp_micros",
             F::cast(DataType::Timestamp(
@@ -1102,7 +1277,8 @@ pub(super) fn list_built_in_datetime_functions() -> Vec<(&'static str, ScalarFun
         ("timestampdiff", F::custom(datediff)),
         ("timestamp_diff", F::custom(datediff)),
         ("to_date", F::custom(to_date)),
-        ("to_time", F::var_arg(to_time)),
+        ("to_time", F::custom(to_time)),
+        ("try_to_time", F::custom(try_to_time)),
         (
             "to_timestamp",
             F::custom(|input| {
diff --git a/crates/sail-plan/src/function/scalar/hash.rs b/crates/sail-plan/src/function/scalar/hash.rs
index cba032cf2..4f27b950c 100644
--- a/crates/sail-plan/src/function/scalar/hash.rs
+++ b/crates/sail-plan/src/function/scalar/hash.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use datafusion::functions::expr_fn;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{expr, ScalarUDF};
+use datafusion_expr::{ScalarUDF, expr};
 use datafusion_spark::function::hash::expr_fn as hash_fn;
 use datafusion_spark::function::hash::xxhash64::SparkXxhash64;
 use datafusion_spark::function::math::hex::SparkHex;
@@ -24,7 +24,7 @@ fn sha2(input: ScalarFunctionInput) -> PlanResult {
         _ => {
             return Err(crate::error::PlanError::invalid(
                 "The second argument of sha2 must be a literal integer.",
-            ))
+            ));
         }
     };
     let hex = expr::Expr::ScalarFunction(expr::ScalarFunction {
diff --git a/crates/sail-plan/src/function/scalar/json.rs b/crates/sail-plan/src/function/scalar/json.rs
index 27dcb82ff..b7b133827 100644
--- a/crates/sail-plan/src/function/scalar/json.rs
+++ b/crates/sail-plan/src/function/scalar/json.rs
@@ -1,15 +1,15 @@
 use chumsky::prelude::*;
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{cast, expr, lit, when, Expr, ScalarUDF};
+use datafusion_expr::{Expr, ScalarUDF, cast, expr, lit, when};
 use datafusion_functions::unicode::expr_fn as unicode_fn;
 use datafusion_spark::expr_fn::json_tuple as df_json_tuple;
 use sail_common_datafusion::literal::LiteralEvaluator;
 use sail_function::scalar::array::spark_array::SparkArray;
 use sail_function::scalar::explode::{Explode, ExplodeKind};
 use sail_function::scalar::json::{
-    json_as_text_udf, json_length_udf, json_object_keys_udf, to_json_udf, SparkFromJson,
-    SparkSchemaOfJson,
+    SparkFromJson, SparkSchemaOfJson, json_as_text_udf, json_length_udf, json_object_keys_udf,
+    to_json_udf,
 };
 
 use crate::error::{PlanError, PlanResult};
@@ -34,8 +34,8 @@ fn parse_json_path(path: &str) -> Option> {
 /// `spark_variant_path_parser` used by `variant_get`. Each path segment maps to
 /// one literal `Expr`: a `Utf8` literal for an object key, an `Int64` literal
 /// for an array index. Any path Spark rejects fails the parse (→ `None`).
-fn json_path_parser<'src>(
-) -> impl Parser<'src, &'src str, Vec, extra::Err>> {
+fn json_path_parser<'src>()
+-> impl Parser<'src, &'src str, Vec, extra::Err>> {
     // `.key` — a run of characters other than `.` or `[`.
     let dot_key = just('.')
         .ignore_then(
@@ -84,12 +84,14 @@ fn get_json_object(expr: expr::Expr, path: expr::Expr) -> PlanResult
             }
         }
         // FIXME: json_as_text_udf for array of paths with subpaths is not implemented, so only top level keys supported
-        _ => vec![when(
-            path.clone().like(lit("$.%")),
-            unicode_fn::substr(path, lit(3)),
-        )
-        .when(lit(true), lit(""))
-        .end()?],
+        _ => vec![
+            when(
+                path.clone().like(lit("$.%")),
+                unicode_fn::substr(path, lit(3)),
+            )
+            .when(lit(true), lit(""))
+            .end()?,
+        ],
     };
     let mut args = Vec::with_capacity(1 + paths.len());
     args.push(expr);
@@ -157,7 +159,7 @@ fn json_tuple(input: ScalarFunctionInput) -> PlanResult {
             _ => {
                 return Err(PlanError::invalid(
                     "json_tuple field names must be string literals",
-                ))
+                ));
             }
         }
     }
diff --git a/crates/sail-plan/src/function/scalar/lambda.rs b/crates/sail-plan/src/function/scalar/lambda.rs
index 5387f4537..14e06f41b 100644
--- a/crates/sail-plan/src/function/scalar/lambda.rs
+++ b/crates/sail-plan/src/function/scalar/lambda.rs
@@ -1,10 +1,10 @@
 use std::sync::{Arc, LazyLock};
 
+use datafusion_common::ScalarValue;
 use datafusion_common::arrow::datatypes::FieldRef;
 use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
-use datafusion_common::ScalarValue;
 use datafusion_expr::expr::{HigherOrderFunction, Lambda, LambdaVariable};
-use datafusion_expr::{expr, lit, HigherOrderUDF, LambdaParametersProgress, ValueOrLambda};
+use datafusion_expr::{HigherOrderUDF, LambdaParametersProgress, ValueOrLambda, expr, lit};
 use datafusion_functions_nested::expr_fn;
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::array::spark_array_aggregate::SparkArrayAggregate;
@@ -82,7 +82,7 @@ pub(crate) fn get_lambda_parameters(
         other => {
             return Err(PlanError::internal(format!(
                 "not a higher-order function: {other}"
-            )))
+            )));
         }
     };
     match udf.lambda_parameters(0, fields)? {
@@ -203,15 +203,15 @@ fn forall(input: ScalarFunctionInput) -> PlanResult {
 /// so anything that is not a bare `Expr::Lambda` falls back to the lenient path
 /// (no arity check here) until expr matching is improved more broadly.
 fn expect_lambda_arity(role: &str, expr: &expr::Expr, arity: usize) -> PlanResult<()> {
-    if let expr::Expr::Lambda(lambda) = expr {
-        if lambda.params.len() != arity {
-            // Mirrors Spark's `INVALID_LAMBDA_FUNCTION_CALL.NUM_ARGS_MISMATCH`
-            // wording, naming no function (`aggregate`/`reduce` share this builder).
-            return Err(PlanError::AnalysisError(format!(
-                "Invalid lambda function call. The {role} lambda function expects {arity} arguments, but got {}",
-                lambda.params.len()
-            )));
-        }
+    if let expr::Expr::Lambda(lambda) = expr
+        && lambda.params.len() != arity
+    {
+        // Mirrors Spark's `INVALID_LAMBDA_FUNCTION_CALL.NUM_ARGS_MISMATCH`
+        // wording, naming no function (`aggregate`/`reduce` share this builder).
+        return Err(PlanError::AnalysisError(format!(
+            "Invalid lambda function call. The {role} lambda function expects {arity} arguments, but got {}",
+            lambda.params.len()
+        )));
     }
     Ok(())
 }
@@ -283,7 +283,7 @@ fn array_sort_spark(array: expr::Expr, asc: expr::Expr) -> PlanResult {
             return Err(PlanError::invalid(format!(
                 "Invalid asc value for array_sort_spark: {asc}"
-            )))
+            )));
         }
     };
     Ok(expr_fn::array_sort(array, sort, nulls))
diff --git a/crates/sail-plan/src/function/scalar/map.rs b/crates/sail-plan/src/function/scalar/map.rs
index 942f2a07d..fb2eafa0a 100644
--- a/crates/sail-plan/src/function/scalar/map.rs
+++ b/crates/sail-plan/src/function/scalar/map.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use datafusion::arrow::datatypes::{DataType, FieldRef, Fields};
 use datafusion::functions_nested::expr_fn;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{cast, expr, lit, ExprSchemable, ScalarUDF};
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, lit};
 use datafusion_spark::function::map::map_from_arrays::MapFromArrays;
 use datafusion_spark::function::map::map_from_entries::MapFromEntries;
 use sail_common_datafusion::utils::items::ItemTaker;
@@ -24,10 +24,11 @@ fn map(input: ScalarFunctionInput) -> PlanResult {
     }
 
     let schema = input.function_context.schema;
-    let (keys, values): (Vec<_>, Vec<_>) = input
-        .arguments
-        .chunks(2)
-        .map(|key_value| (key_value[0].clone(), key_value[1].clone()))
+    let (pairs, remainder) = input.arguments.as_chunks::<2>();
+    debug_assert!(remainder.is_empty());
+    let (keys, values): (Vec<_>, Vec<_>) = pairs
+        .iter()
+        .map(|[key, value]| (key.clone(), value.clone()))
         .unzip();
     let value_contains_null = values.iter().try_fold(false, |nullable, value| {
         Ok::<_, PlanError>(nullable || value.nullable(schema.as_ref())?)
@@ -246,20 +247,21 @@ fn map_concat(input: ScalarFunctionInput) -> PlanResult {
     // Wrap the result with CASE to handle NULLs:
     // CASE WHEN arg1 IS NULL OR arg2 IS NULL OR ... THEN NULL ELSE result END
     // We already checked that arguments is not empty, so reduce will always return Some
-    if let Some(null_check) = input
+    match input
         .arguments
         .iter()
         .map(|arg| arg.clone().is_null())
         .reduce(|a, b| a.or(b))
     {
-        Ok(Expr::Case(expr::Case {
+        Some(null_check) => Ok(Expr::Case(expr::Case {
             expr: None,
             when_then_expr: vec![(Box::new(null_check), Box::new(lit(ScalarValue::Null)))],
             else_expr: Some(Box::new(result)),
-        }))
-    } else {
-        // This should never happen because we checked arguments is not empty
-        Ok(result)
+        })),
+        _ => {
+            // This should never happen because we checked arguments is not empty
+            Ok(result)
+        }
     }
 }
 
diff --git a/crates/sail-plan/src/function/scalar/math.rs b/crates/sail-plan/src/function/scalar/math.rs
index 2143f3d77..fb0bdf9ac 100644
--- a/crates/sail-plan/src/function/scalar/math.rs
+++ b/crates/sail-plan/src/function/scalar/math.rs
@@ -1,11 +1,11 @@
 use std::sync::Arc;
 
-use datafusion::arrow::datatypes::{i256, DataType, IntervalUnit, TimeUnit};
+use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit, i256};
 use datafusion::arrow::error::ArrowError;
 use datafusion::functions::expr_fn;
 use datafusion_common::{DFSchemaRef, ScalarValue};
 use datafusion_expr::{
-    cast, expr, lit, try_cast, BinaryExpr, Expr, ExprSchemable, Operator, ScalarUDF,
+    BinaryExpr, Expr, ExprSchemable, Operator, ScalarUDF, cast, expr, lit, try_cast,
 };
 use datafusion_spark::function::math::expr_fn as math_fn;
 use half::f16;
@@ -425,7 +425,11 @@ fn eulers_constant() -> Expr {
 }
 
 fn ceil_floor(input: ScalarFunctionInput, name: &str) -> PlanResult {
-    let ScalarFunctionInput { arguments, .. } = input;
+    let ScalarFunctionInput {
+        arguments,
+        function_context,
+    } = input;
+    let ansi_mode = function_context.plan_config.ansi_mode;
     // DataFusion bug: `ReturnTypeArgs.scalar_arguments` is None if scalar argument is nested
     let arguments = if arguments.len() == 2 {
         let (arg, target_scale) = arguments.two()?;
@@ -487,9 +491,9 @@ fn ceil_floor(input: ScalarFunctionInput, name: &str) -> PlanResult {
         arguments
     };
     let func = if matches!(name.to_lowercase().trim(), "ceil") {
-        Arc::new(ScalarUDF::from(SparkCeil::new()))
+        Arc::new(ScalarUDF::from(SparkCeil::new(ansi_mode)))
     } else {
-        Arc::new(ScalarUDF::from(SparkFloor::new()))
+        Arc::new(ScalarUDF::from(SparkFloor::new(ansi_mode)))
     };
     Ok(Expr::ScalarFunction(expr::ScalarFunction {
         func,
diff --git a/crates/sail-plan/src/function/scalar/misc.rs b/crates/sail-plan/src/function/scalar/misc.rs
index b2db79b0c..53c09a513 100644
--- a/crates/sail-plan/src/function/scalar/misc.rs
+++ b/crates/sail-plan/src/function/scalar/misc.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use arrow::datatypes::DataType;
 use datafusion::functions::expr_fn;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{cast, expr, lit, when, ExprSchemable, Operator, ScalarUDF};
+use datafusion_expr::{ExprSchemable, Operator, ScalarUDF, cast, expr, lit, when};
 use datafusion_spark::function::bitmap::expr_fn as bitmap_fn;
 use sail_catalog::manager::CatalogManager;
 use sail_catalog::utils::quote_namespace_if_needed;
@@ -33,7 +33,7 @@ fn assert_true(input: ScalarFunctionInput) -> PlanResult {
         let col = arguments.one()?;
         (
             // Need to do this order to avoid the "value used after being moved" error.
-            lit(ScalarValue::Utf8(Some(format!("'{}' is not true!", &col)))),
+            lit(ScalarValue::Utf8(Some(format!("'{}' is not true!", col)))),
             col,
         )
     } else if arguments.len() == 2 {
@@ -144,7 +144,7 @@ fn theta_union(input: ScalarFunctionInput) -> PlanResult {
         count => {
             return Err(PlanError::invalid(format!(
                 "theta_union requires 2 or 3 arguments, got {count}"
-            )))
+            )));
         }
     };
     Ok(ScalarUDF::from(ThetaUnionFunction::new()).call(arguments))
@@ -168,7 +168,7 @@ fn hll_union(input: ScalarFunctionInput) -> PlanResult {
         count => {
             return Err(PlanError::invalid(format!(
                 "hll_union requires 2 or 3 arguments, got {count}"
-            )))
+            )));
         }
     };
     Ok(ScalarUDF::from(HllUnionFunction::new()).call(arguments))
@@ -186,6 +186,7 @@ pub(super) fn list_built_in_misc_functions() -> Vec<(&'static str, ScalarFunctio
         ("bitmap_count", F::unary(bitmap_fn::bitmap_count)),
         ("current_catalog", F::custom(current_catalog)),
         ("current_database", F::custom(current_database)),
+        ("current_path", F::unknown("current_path")),
         ("current_schema", F::custom(current_database)),
         ("current_user", F::custom(current_user)),
         ("from_avro", F::unknown("from_avro")),
@@ -206,6 +207,72 @@ pub(super) fn list_built_in_misc_functions() -> Vec<(&'static str, ScalarFunctio
             F::udf(ThetaSketchEstimateFunction::new()),
         ),
         ("theta_union", F::custom(theta_union)),
+        (
+            "tuple_difference_double",
+            F::unknown("tuple_difference_double"),
+        ),
+        (
+            "tuple_difference_integer",
+            F::unknown("tuple_difference_integer"),
+        ),
+        (
+            "tuple_difference_theta_double",
+            F::unknown("tuple_difference_theta_double"),
+        ),
+        (
+            "tuple_difference_theta_integer",
+            F::unknown("tuple_difference_theta_integer"),
+        ),
+        (
+            "tuple_intersection_double",
+            F::unknown("tuple_intersection_double"),
+        ),
+        (
+            "tuple_intersection_integer",
+            F::unknown("tuple_intersection_integer"),
+        ),
+        (
+            "tuple_intersection_theta_double",
+            F::unknown("tuple_intersection_theta_double"),
+        ),
+        (
+            "tuple_intersection_theta_integer",
+            F::unknown("tuple_intersection_theta_integer"),
+        ),
+        (
+            "tuple_sketch_estimate_double",
+            F::unknown("tuple_sketch_estimate_double"),
+        ),
+        (
+            "tuple_sketch_estimate_integer",
+            F::unknown("tuple_sketch_estimate_integer"),
+        ),
+        (
+            "tuple_sketch_summary_double",
+            F::unknown("tuple_sketch_summary_double"),
+        ),
+        (
+            "tuple_sketch_summary_integer",
+            F::unknown("tuple_sketch_summary_integer"),
+        ),
+        (
+            "tuple_sketch_theta_double",
+            F::unknown("tuple_sketch_theta_double"),
+        ),
+        (
+            "tuple_sketch_theta_integer",
+            F::unknown("tuple_sketch_theta_integer"),
+        ),
+        ("tuple_union_double", F::unknown("tuple_union_double")),
+        ("tuple_union_integer", F::unknown("tuple_union_integer")),
+        (
+            "tuple_union_theta_double",
+            F::unknown("tuple_union_theta_double"),
+        ),
+        (
+            "tuple_union_theta_integer",
+            F::unknown("tuple_union_theta_integer"),
+        ),
         (
             "input_file_block_length",
             F::unknown("input_file_block_length"),
diff --git a/crates/sail-plan/src/function/scalar/mod.rs b/crates/sail-plan/src/function/scalar/mod.rs
index 6d5670a11..9518a31d7 100644
--- a/crates/sail-plan/src/function/scalar/mod.rs
+++ b/crates/sail-plan/src/function/scalar/mod.rs
@@ -21,6 +21,7 @@ mod string;
 mod r#struct;
 mod url;
 mod variant;
+mod vector;
 mod xml;
 
 pub(super) fn list_built_in_scalar_functions() -> Vec<(&'static str, ScalarFunction)> {
@@ -44,6 +45,7 @@ pub(super) fn list_built_in_scalar_functions() -> Vec<(&'static str, ScalarFunct
     output.extend(r#struct::list_built_in_struct_functions());
     output.extend(url::list_built_in_url_functions());
     output.extend(variant::list_built_in_variant_functions());
+    output.extend(vector::list_built_in_vector_functions());
     output.extend(xml::list_built_in_xml_functions());
     output
 }
diff --git a/crates/sail-plan/src/function/scalar/predicate.rs b/crates/sail-plan/src/function/scalar/predicate.rs
index e5743a3dd..63dbc6aa2 100644
--- a/crates/sail-plan/src/function/scalar/predicate.rs
+++ b/crates/sail-plan/src/function/scalar/predicate.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use datafusion::functions::expr_fn;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{expr, lit, not, Operator, ScalarUDF};
+use datafusion_expr::{Operator, ScalarUDF, expr, lit, not};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::predicate::rewrite_like_pattern::RewriteLikePatternFunc;
 
diff --git a/crates/sail-plan/src/function/scalar/string.rs b/crates/sail-plan/src/function/scalar/string.rs
index 682fd2daa..8dba1ad7f 100644
--- a/crates/sail-plan/src/function/scalar/string.rs
+++ b/crates/sail-plan/src/function/scalar/string.rs
@@ -4,13 +4,16 @@ use datafusion::functions::regex::expr_fn as regex_fn;
 use datafusion::functions::regex::regexpcount::RegexpCountFunc;
 use datafusion::functions::regex::regexpinstr::RegexpInstrFunc;
 use datafusion_common::{DFSchema, ScalarValue};
-use datafusion_expr::{cast, expr, lit, try_cast, when, ExprSchemable, ScalarUDF};
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, lit, try_cast, when};
 use datafusion_functions_nested::expr_fn::array_element;
 use datafusion_spark::function::math::expr_fn as math_fn;
 use datafusion_spark::function::string::elt::SparkElt;
 use datafusion_spark::function::string::expr_fn as string_fn;
 use datafusion_spark::function::string::format_string::FormatStringFunc;
+use datafusion_spark::function::string::length::SparkLengthFunc;
+use regex_syntax::hir::Look;
 use sail_common_datafusion::utils::items::ItemTaker;
+use sail_function::scalar::spark_to_string::SparkToUtf8;
 use sail_function::scalar::string::format_number::FormatNumber;
 use sail_function::scalar::string::levenshtein::Levenshtein;
 use sail_function::scalar::string::make_valid_utf8::MakeValidUtf8;
@@ -19,6 +22,7 @@ use sail_function::scalar::string::soundex::Soundex;
 use sail_function::scalar::string::spark_base64::{SparkBase64, SparkUnbase64};
 use sail_function::scalar::string::spark_concat_ws::SparkConcatWs;
 use sail_function::scalar::string::spark_encode_decode::{SparkDecode, SparkEncode};
+use sail_function::scalar::string::spark_length::{SparkBitLength, SparkOctetLength};
 use sail_function::scalar::string::spark_mask::SparkMask;
 use sail_function::scalar::string::spark_quote::SparkQuote;
 use sail_function::scalar::string::spark_regexp_extract_all::{
@@ -34,8 +38,39 @@ use crate::error::{PlanError, PlanResult};
 use crate::function::common::{ScalarFunction, ScalarFunctionInput};
 use crate::function::scalar::datetime::date_format;
 
+fn is_single_capture_extract(pattern: &expr::Expr, replacement: &expr::Expr) -> bool {
+    let (expr::Expr::Literal(pattern, _), expr::Expr::Literal(replacement, _)) =
+        (pattern, replacement)
+    else {
+        return false;
+    };
+    let (Some(pattern), Some("$1")) = (
+        pattern.try_as_str().flatten(),
+        replacement.try_as_str().flatten(),
+    ) else {
+        return false;
+    };
+    if !pattern.starts_with('^') {
+        return false;
+    }
+    let Some(short_pattern) = pattern.strip_suffix(".*$") else {
+        return false;
+    };
+
+    // Absolute start anchoring makes global and single replacement equivalent.
+    regex_syntax::parse(short_pattern).is_ok_and(|pattern| {
+        let properties = pattern.properties();
+        properties.look_set_prefix().contains(Look::Start)
+            && properties.explicit_captures_len() == 1
+    })
+}
+
 fn regexp_replace(string: expr::Expr, pattern: expr::Expr, replacement: expr::Expr) -> expr::Expr {
-    regex_fn::regexp_replace(string, pattern, replacement, Some(lit("g")))
+    if is_single_capture_extract(&pattern, &replacement) {
+        regex_fn::regexp_replace(string, pattern, lit("${1}"), None)
+    } else {
+        regex_fn::regexp_replace(string, pattern, replacement, Some(lit("g")))
+    }
 }
 
 fn regexp_substr(input: ScalarFunctionInput) -> PlanResult {
@@ -162,12 +197,53 @@ fn contains(input: ScalarFunctionInput) -> PlanResult {
     in_str_str_out_bool(expr_fn::contains)(input)
 }
 
+/// Spark measures the character length of string data and the byte length of binary data,
+/// so binary must reach the function as-is. Any other type is measured as its string form,
+/// via the Spark-compatible cast rather than the Arrow one, which renders a timestamp with
+/// a time zone suffix. Collections have no string form in Spark and are rejected.
+fn length_argument(input: ScalarFunctionInput, name: &str) -> PlanResult {
+    let ScalarFunctionInput {
+        arguments,
+        function_context,
+    } = input;
+    let arg = arguments.one()?;
+    let data_type = arg.get_type(function_context.schema)?;
+    match data_type {
+        DataType::Utf8
+        | DataType::LargeUtf8
+        | DataType::Utf8View
+        | DataType::Binary
+        | DataType::LargeBinary
+        | DataType::BinaryView => Ok(arg),
+        DataType::FixedSizeBinary(_) => Ok(cast(arg, DataType::Binary)),
+        DataType::Null => Ok(cast(arg, DataType::Utf8)),
+        DataType::List(_)
+        | DataType::LargeList(_)
+        | DataType::ListView(_)
+        | DataType::LargeListView(_)
+        | DataType::FixedSizeList(_, _)
+        | DataType::Struct(_)
+        | DataType::Map(_, _)
+        | DataType::Union(_, _) => Err(PlanError::invalid(format!(
+            "`{name}` does not support {data_type} input"
+        ))),
+        _ => Ok(ScalarUDF::from(SparkToUtf8::new()).call(vec![arg])),
+    }
+}
+
+fn length(input: ScalarFunctionInput) -> PlanResult {
+    let arg = length_argument(input, "length")?;
+    Ok(ScalarUDF::from(SparkLengthFunc::new()).call(vec![arg]))
+}
+
 fn bit_length(input: ScalarFunctionInput) -> PlanResult {
-    in_str_out_i32(expr_fn::bit_length)(input)
+    let arg = length_argument(input, "bit_length")?;
+    Ok(ScalarUDF::from(SparkBitLength::new()).call(vec![arg]))
 }
 
 fn octet_length(input: ScalarFunctionInput) -> PlanResult {
-    in_str_out_i32(expr_fn::octet_length)(input)
+    let arg = length_argument(input, "octet_length")?;
+    Ok(ScalarUDF::from(SparkOctetLength::new()).call(vec![arg]))
 }
 
 fn ascii(input: ScalarFunctionInput) -> PlanResult {
@@ -251,7 +327,8 @@ fn to_char(input: ScalarFunctionInput) -> PlanResult {
         .map_err(|_| PlanError::invalid("to_char requires 2 arguments"))?;
     match value.get_type(function_context.schema)? {
         DataType::Date32 | DataType::Date64 | DataType::Timestamp(_, _) => {
-            Ok(date_format(value, format))
+            let timezone = function_context.plan_config.session_timezone.clone();
+            Ok(date_format(value, format, timezone.to_string()))
         }
         DataType::Binary
         | DataType::LargeBinary
@@ -297,8 +374,8 @@ pub(super) fn list_built_in_string_functions() -> Vec<(&'static str, ScalarFunct
         ("bit_length", F::custom(bit_length)),
         ("btrim", F::var_arg(expr_fn::btrim)),
         ("char", F::unary(expr_fn::chr)),
-        ("char_length", F::unary(expr_fn::char_length)),
-        ("character_length", F::unary(expr_fn::char_length)),
+        ("char_length", F::custom(length)),
+        ("character_length", F::custom(length)),
         ("chr", F::unary(expr_fn::chr)),
         ("collate", F::unknown("collate")),
         ("collation", F::unknown("collation")),
@@ -316,8 +393,8 @@ pub(super) fn list_built_in_string_functions() -> Vec<(&'static str, ScalarFunct
         ("is_valid_utf8", F::custom(is_valid_utf8)),
         ("lcase", F::custom(lower)),
         ("left", F::binary(expr_fn::left)),
-        ("len", F::unary(expr_fn::length)),
-        ("length", F::unary(expr_fn::length)),
+        ("len", F::custom(length)),
+        ("length", F::custom(length)),
         ("levenshtein", F::udf(Levenshtein::new())),
         ("locate", F::custom(position)),
         ("lower", F::custom(lower)),
diff --git a/crates/sail-plan/src/function/scalar/struct.rs b/crates/sail-plan/src/function/scalar/struct.rs
index a45179537..b2aaaadc2 100644
--- a/crates/sail-plan/src/function/scalar/struct.rs
+++ b/crates/sail-plan/src/function/scalar/struct.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use datafusion::functions::expr_fn;
-use datafusion_expr::{expr, Expr, ScalarUDF};
+use datafusion_expr::{Expr, ScalarUDF, expr};
 use sail_function::scalar::struct_function::StructFunction;
 
 use crate::error::{PlanError, PlanResult};
diff --git a/crates/sail-plan/src/function/scalar/variant.rs b/crates/sail-plan/src/function/scalar/variant.rs
index 28900ca33..b5ca8dea6 100644
--- a/crates/sail-plan/src/function/scalar/variant.rs
+++ b/crates/sail-plan/src/function/scalar/variant.rs
@@ -11,6 +11,7 @@ pub(super) fn list_built_in_variant_functions() -> Vec<(&'static str, ScalarFunc
     use crate::function::common::ScalarFunctionBuilder as F;
 
     vec![
+        ("is_valid_variant", F::unknown("is_valid_variant")),
         ("is_variant_null", F::udf(SparkIsVariantNullUdf::new())),
         ("parse_json", F::udf(SparkParseJson::new(false))),
         ("schema_of_variant", F::udf(SparkSchemaOfVariantUdf::new())),
diff --git a/crates/sail-plan/src/function/scalar/vector.rs b/crates/sail-plan/src/function/scalar/vector.rs
new file mode 100644
index 000000000..49d734a06
--- /dev/null
+++ b/crates/sail-plan/src/function/scalar/vector.rs
@@ -0,0 +1,16 @@
+use crate::function::common::ScalarFunction;
+
+pub(super) fn list_built_in_vector_functions() -> Vec<(&'static str, ScalarFunction)> {
+    use crate::function::common::ScalarFunctionBuilder as F;
+
+    vec![
+        (
+            "vector_cosine_similarity",
+            F::unknown("vector_cosine_similarity"),
+        ),
+        ("vector_inner_product", F::unknown("vector_inner_product")),
+        ("vector_l2_distance", F::unknown("vector_l2_distance")),
+        ("vector_norm", F::unknown("vector_norm")),
+        ("vector_normalize", F::unknown("vector_normalize")),
+    ]
+}
diff --git a/crates/sail-plan/src/function/scalar/xml.rs b/crates/sail-plan/src/function/scalar/xml.rs
index 7d46be4d1..287167ca2 100644
--- a/crates/sail-plan/src/function/scalar/xml.rs
+++ b/crates/sail-plan/src/function/scalar/xml.rs
@@ -1,5 +1,7 @@
-use datafusion_expr::{expr, Expr, ScalarUDF};
+use datafusion_expr::{Expr, ScalarUDF, expr};
+use sail_common_datafusion::literal::LiteralEvaluator;
 use sail_common_datafusion::utils::items::ItemTaker;
+use sail_function::scalar::xml::from_xml::SparkFromXml;
 use sail_function::scalar::xml::to_xml::SparkToXml;
 use sail_function::scalar::xml::xpath::Xpath;
 use sail_function::scalar::xml::xpath_typed::{XpathTyped, XpathTypedKind};
@@ -33,11 +35,35 @@ fn to_xml(input: ScalarFunctionInput) -> PlanResult {
     Ok(udf.call(input.arguments))
 }
 
+fn from_xml(
+    ScalarFunctionInput {
+        mut arguments,
+        function_context,
+    }: ScalarFunctionInput,
+) -> PlanResult {
+    let tz = function_context.plan_config.session_timezone.clone();
+    if arguments.len() >= 2 && !matches!(&arguments[1], expr::Expr::Literal(_, _)) {
+        let evaluator = LiteralEvaluator::new();
+        if let Ok(scalar) = evaluator.evaluate(&arguments[1]) {
+            let scalar = match scalar {
+                datafusion_common::ScalarValue::Utf8View(v)
+                | datafusion_common::ScalarValue::LargeUtf8(v) => {
+                    datafusion_common::ScalarValue::Utf8(v)
+                }
+                other => other,
+            };
+            arguments[1] = expr::Expr::Literal(scalar, None);
+        }
+    }
+    let udf = ScalarUDF::from(SparkFromXml::new(tz));
+    Ok(udf.call(arguments))
+}
+
 pub(super) fn list_built_in_xml_functions() -> Vec<(&'static str, ScalarFunction)> {
     use crate::function::common::ScalarFunctionBuilder as F;
 
     vec![
-        ("from_xml", F::unknown("from_xml")),
+        ("from_xml", F::custom(from_xml)),
         ("schema_of_xml", F::unknown("schema_of_xml")),
         ("to_xml", F::custom(to_xml)),
         ("xpath", F::custom(xpath)),
diff --git a/crates/sail-plan/src/function/table/range.rs b/crates/sail-plan/src/function/table/range.rs
index c2d243fb0..01ae060cd 100644
--- a/crates/sail-plan/src/function/table/range.rs
+++ b/crates/sail-plan/src/function/table/range.rs
@@ -5,8 +5,8 @@ use std::sync::Arc;
 use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::catalog::{Session, TableFunctionImpl, TableProvider};
 use datafusion::physical_plan::ExecutionPlan;
-use datafusion_common::{exec_err, Result};
-use datafusion_expr::{logical_plan, Expr, LogicalPlan, TableType, UserDefinedLogicalNodeCore};
+use datafusion_common::{Result, exec_err};
+use datafusion_expr::{Expr, LogicalPlan, TableType, UserDefinedLogicalNodeCore, logical_plan};
 use sail_common_datafusion::literal::{LiteralEvaluator, LiteralValue};
 use sail_logical_plan::range::RangeNode;
 use sail_physical_plan::range::RangeExec;
@@ -101,7 +101,7 @@ impl TableFunctionImpl for RangeTableFunction {
                 return exec_err!(
                     "expected 1 to 4 arguments for range, but got {}",
                     args.len()
-                )
+                );
             }
         };
         let node = RangeNode::try_new("id".to_string(), start, end, step, num_partitions)?;
diff --git a/crates/sail-plan/src/function/window.rs b/crates/sail-plan/src/function/window.rs
index d441ceb41..ca53f1e82 100644
--- a/crates/sail-plan/src/function/window.rs
+++ b/crates/sail-plan/src/function/window.rs
@@ -13,9 +13,9 @@ use datafusion::functions_window::nth_value::{first_value_udwf, last_value_udwf,
 use datafusion::functions_window::rank::{dense_rank_udwf, percent_rank_udwf, rank_udwf};
 use datafusion::functions_window::row_number::row_number_udwf;
 use datafusion_common::ScalarValue;
-use datafusion_expr::expr::WindowFunctionParams;
+use datafusion_expr::expr::{NullTreatment, WindowFunctionParams};
 use datafusion_expr::{
-    cast, expr, lit, when, AggregateUDF, ExprSchemable, WindowFrame, WindowFunctionDefinition,
+    AggregateUDF, ExprSchemable, WindowFrame, WindowFunctionDefinition, cast, expr, lit, when,
 };
 use datafusion_spark::function::aggregate::try_sum::SparkTrySum;
 use lazy_static::lazy_static;
@@ -38,13 +38,13 @@ use sail_function::aggregate::theta_sketch::{
     ThetaIntersectionAggFunction, ThetaSketchAggFunction, ThetaUnionAggFunction,
 };
 use sail_function::aggregate::try_avg::TryAvgFunction;
-use sail_function::window::spark_ntile_udwf;
+use sail_function::window::{spark_first_value_udwf, spark_last_value_udwf, spark_ntile_udwf};
 
 use crate::error::{PlanError, PlanResult};
 use crate::function::common::{
-    count_min_sketch_args, get_arguments_and_null_treatment, get_null_treatment,
-    hll_args_with_default_lg, hll_union_args_with_default_allow_different_lg,
-    theta_args_with_default_lg, WinFunction, WinFunctionInput,
+    WinFunction, WinFunctionInput, count_min_sketch_args, get_arguments_and_null_treatment,
+    get_null_treatment, hll_args_with_default_lg, hll_union_args_with_default_allow_different_lg,
+    theta_args_with_default_lg,
 };
 use crate::function::transform_count_star_wildcard_expr;
 
@@ -135,8 +135,15 @@ fn first_value(input: WinFunctionInput) -> PlanResult {
         function_context: _,
     } = input;
     let (args, null_treatment) = get_arguments_and_null_treatment(arguments, ignore_nulls)?;
+    // DataFusion physical proto drops IGNORE NULLS for WindowUDFExpr, so use a
+    // Sail UDWF with the null treatment encoded in the function state.
+    let fun = if matches!(null_treatment, Some(NullTreatment::IgnoreNulls)) {
+        WindowFunctionDefinition::WindowUDF(spark_first_value_udwf(true))
+    } else {
+        WindowFunctionDefinition::WindowUDF(first_value_udwf())
+    };
     Ok(expr::Expr::WindowFunction(Box::new(expr::WindowFunction {
-        fun: WindowFunctionDefinition::WindowUDF(first_value_udwf()),
+        fun,
         params: WindowFunctionParams {
             args,
             partition_by,
@@ -160,8 +167,13 @@ fn last_value(input: WinFunctionInput) -> PlanResult {
         function_context: _,
     } = input;
     let (args, null_treatment) = get_arguments_and_null_treatment(arguments, ignore_nulls)?;
+    let fun = if matches!(null_treatment, Some(NullTreatment::IgnoreNulls)) {
+        WindowFunctionDefinition::WindowUDF(spark_last_value_udwf(true))
+    } else {
+        WindowFunctionDefinition::WindowUDF(last_value_udwf())
+    };
     Ok(expr::Expr::WindowFunction(Box::new(expr::WindowFunction {
-        fun: WindowFunctionDefinition::WindowUDF(last_value_udwf()),
+        fun,
         params: WindowFunctionParams {
             args,
             partition_by,
@@ -429,18 +441,19 @@ fn count_if(input: WinFunctionInput) -> PlanResult {
     } = input;
     match arguments.len() {
         1 => {
-            let filter = arguments
+            let predicate = arguments
                 .first()
                 .ok_or_else(|| PlanError::invalid("`count_if` requires 1 argument"))?
                 .clone();
+            let arg = when(predicate, lit(0)).otherwise(lit(ScalarValue::Int32(None)))?;
             Ok(expr::Expr::WindowFunction(Box::new(expr::WindowFunction {
                 fun: WindowFunctionDefinition::AggregateUDF(count::count_udaf()),
                 params: WindowFunctionParams {
-                    args: vec![lit(0)],
+                    args: vec![arg],
                     partition_by,
                     order_by,
                     window_frame,
-                    filter: Some(Box::new(filter)),
+                    filter: None,
                     null_treatment: get_null_treatment(ignore_nulls),
                     distinct,
                 },
@@ -474,7 +487,7 @@ fn collect_set(input: WinFunctionInput) -> PlanResult {
             order_by,
             window_frame,
             filter: null_filter,
-            null_treatment: None,
+            null_treatment: get_null_treatment(Some(true)),
             distinct: true,
         },
     })))
@@ -809,3 +822,7 @@ pub(crate) fn get_built_in_window_function(name: &str) -> PlanResult impl Iterator {
+    BUILT_IN_WINDOW_FUNCTIONS.keys().copied()
+}
diff --git a/crates/sail-plan/src/lib.rs b/crates/sail-plan/src/lib.rs
index c7e4f51a9..bb87e6d39 100644
--- a/crates/sail-plan/src/lib.rs
+++ b/crates/sail-plan/src/lib.rs
@@ -1,18 +1,18 @@
 use std::sync::Arc;
 
 use datafusion::dataframe::DataFrame;
-use datafusion::physical_plan::{displayable, ExecutionPlan};
+use datafusion::physical_plan::{ExecutionPlan, displayable};
 use datafusion::prelude::SessionContext;
-use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan};
 use datafusion_common::Result;
+use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan};
 use datafusion_expr::LogicalPlan;
 use sail_common::spec;
 use sail_common_datafusion::rename::physical_plan::rename_physical_plan;
 
 use crate::config::PlanConfig;
 use crate::error::PlanResult;
-use crate::resolver::plan::NamedPlan;
 use crate::resolver::PlanResolver;
+use crate::resolver::plan::NamedPlan;
 use crate::streaming::rewriter::{is_streaming_plan, rewrite_streaming_plan};
 
 pub mod catalog;
diff --git a/crates/sail-plan/src/resolver/command/catalog/table.rs b/crates/sail-plan/src/resolver/command/catalog/table.rs
index 746730aed..469b843f6 100644
--- a/crates/sail-plan/src/resolver/command/catalog/table.rs
+++ b/crates/sail-plan/src/resolver/command/catalog/table.rs
@@ -13,8 +13,8 @@ use sail_common_datafusion::rename::logical_plan::rename_logical_plan;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(in super::super) async fn resolve_catalog_create_table(
diff --git a/crates/sail-plan/src/resolver/command/catalog/view.rs b/crates/sail-plan/src/resolver/command/catalog/view.rs
index 24e2ffd57..48681431e 100644
--- a/crates/sail-plan/src/resolver/command/catalog/view.rs
+++ b/crates/sail-plan/src/resolver/command/catalog/view.rs
@@ -3,8 +3,8 @@ use std::sync::Arc;
 use datafusion_common::TableReference;
 use datafusion_expr::{LogicalPlan, SubqueryAlias};
 use sail_catalog::command::CatalogCommand;
-use sail_catalog::manager::tracker::CatalogLogicalPlanId;
 use sail_catalog::manager::CatalogManager;
+use sail_catalog::manager::tracker::CatalogLogicalPlanId;
 use sail_catalog::provider::{
     CreateTemporaryViewColumnOptions, CreateTemporaryViewOptions, CreateViewColumnOptions,
     CreateViewOptions, DropTemporaryViewOptions, DropViewOptions,
@@ -15,8 +15,8 @@ use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::rename::logical_plan::rename_logical_plan;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(in super::super) async fn resolve_catalog_create_view(
diff --git a/crates/sail-plan/src/resolver/command/checkpoint.rs b/crates/sail-plan/src/resolver/command/checkpoint.rs
new file mode 100644
index 000000000..c0150bf9f
--- /dev/null
+++ b/crates/sail-plan/src/resolver/command/checkpoint.rs
@@ -0,0 +1,36 @@
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::Schema;
+use datafusion_expr::{Extension, LogicalPlan};
+use sail_common::spec;
+use sail_common_datafusion::rename::schema::rename_schema;
+use sail_logical_plan::remote_checkpoint::RemoteCheckpointCommandNode;
+
+use crate::error::PlanResult;
+use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
+
+impl PlanResolver<'_> {
+    pub(super) async fn resolve_command_remote_checkpoint(
+        &self,
+        relation_id: String,
+        input: spec::QueryPlan,
+        state: &mut PlanResolverState,
+    ) -> PlanResult {
+        let input = self.resolve_query_plan(input, state).await?;
+        let names = Self::get_field_names(input.schema(), state)?;
+        let input_schema = input.schema().inner();
+        let renamed = rename_schema(input_schema, &names)?;
+        let logical_schema = Arc::new(Schema::new_with_metadata(
+            renamed.fields().clone(),
+            input_schema.metadata().clone(),
+        ));
+        Ok(LogicalPlan::Extension(Extension {
+            node: Arc::new(RemoteCheckpointCommandNode::new(
+                relation_id,
+                Arc::new(input),
+                logical_schema,
+            )),
+        }))
+    }
+}
diff --git a/crates/sail-plan/src/resolver/command/delete.rs b/crates/sail-plan/src/resolver/command/delete.rs
index ad95cd652..31634321e 100644
--- a/crates/sail-plan/src/resolver/command/delete.rs
+++ b/crates/sail-plan/src/resolver/command/delete.rs
@@ -16,8 +16,8 @@ use sail_common_datafusion::rename::expression::expression_before_rename;
 use sail_common_datafusion::rename::schema::rename_schema;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     /// Resolves the DELETE command.
diff --git a/crates/sail-plan/src/resolver/command/delta.rs b/crates/sail-plan/src/resolver/command/delta.rs
index 124f8cef6..a343cde79 100644
--- a/crates/sail-plan/src/resolver/command/delta.rs
+++ b/crates/sail-plan/src/resolver/command/delta.rs
@@ -4,7 +4,7 @@ use datafusion::arrow::datatypes::{DataType, Field};
 use datafusion_common::{Column, DFSchemaRef};
 use datafusion_expr::expr::FieldMetadata;
 use datafusion_expr::{
-    col, lit, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, ScalarUDF,
+    Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, ScalarUDF, col, lit,
 };
 use sail_catalog::command::CatalogCommand;
 use sail_catalog::error::CatalogError;
@@ -14,20 +14,20 @@ use sail_common::spec;
 use sail_common_datafusion::catalog::{TableColumnStatus, TableKind};
 use sail_common_datafusion::column_features::ColumnFeatures;
 use sail_common_datafusion::datasource::{
-    find_path_in_options, OptionLayer, SinkInfo, SourceInfo, TableFormatRegistry,
+    OptionLayer, SinkInfo, SourceInfo, TableFormatRegistry, find_path_in_options,
 };
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_function::scalar::misc::raise_error::RaiseError;
 use sail_logical_plan::barrier::BarrierNode;
 use sail_logical_plan::check_constraints::{
-    apply_delta_check_constraint_filter, DeltaCheckConstraintExpr, DeltaConstraintViolation,
+    DeltaCheckConstraintExpr, DeltaConstraintViolation, apply_delta_check_constraint_filter,
 };
 
 use super::merge::merge_disambiguate_unqualified_plan_ids;
 use super::write::{TableInfo, WriteColumnMatch};
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 const DELTA_FORMAT: &str = "delta";
 const DELTA_CHECK_CONSTRAINT_PREFIX: &str = "delta.constraints.";
@@ -48,12 +48,12 @@ impl PlanResolver<'_> {
         operation: spec::AlterTableOperation,
         state: &mut PlanResolverState,
     ) -> PlanResult {
-        if let Some(key) = delta_constraint_property_mutation_key(&operation) {
-            if self.alter_table_target_is_delta(&table, if_exists).await? {
-                return Err(PlanError::invalid(format!(
-                    "[DELTA_ADD_CONSTRAINTS] Please use ALTER TABLE ADD CONSTRAINT to add CHECK constraints. Invalid property: {key}"
-                )));
-            }
+        if let Some(key) = delta_constraint_property_mutation_key(&operation)
+            && self.alter_table_target_is_delta(&table, if_exists).await?
+        {
+            return Err(PlanError::invalid(format!(
+                "[DELTA_ADD_CONSTRAINTS] Please use ALTER TABLE ADD CONSTRAINT to add CHECK constraints. Invalid property: {key}"
+            )));
         }
         match operation {
             spec::AlterTableOperation::AddCheckConstraint { name, expression } => {
@@ -298,7 +298,9 @@ impl PlanResolver<'_> {
         let validation = LogicalPlanBuilder::new(scan)
             .filter(Expr::IsNotTrue(Box::new(resolved_expr)))?
             .limit(0, Some(1))?
-            .project(vec![validation_error.alias("__delta_constraint_validation")])?
+            .project(vec![
+                validation_error.alias("__delta_constraint_validation"),
+            ])?
             .build()?;
 
         let command = self.resolve_catalog_command(CatalogCommand::AlterTable {
@@ -364,10 +366,19 @@ impl PlanResolver<'_> {
                 properties.extend(items.clone());
             }
         }
-        let constraints = delta_constraints_from_schema_and_properties(
-            target_schema.fields().iter().map(|field| field.as_ref()),
-            &properties,
-        );
+
+        // MERGE target schemas use opaque field IDs internally. Constraint source text and
+        // diagnostics must use the corresponding user-facing names; resolving that source text
+        // against `target_schema` maps it back to the opaque field safely.
+        let target_field_names = Self::get_field_names(target_schema, state)?;
+        let target_fields = target_schema
+            .fields()
+            .iter()
+            .zip(target_field_names)
+            .map(|(field, name)| field.as_ref().clone().with_name(name))
+            .collect::>();
+        let constraints =
+            delta_constraints_from_schema_and_properties(target_fields.iter(), &properties);
         self.resolve_delta_check_constraints(constraints, target_schema, state)
             .await
     }
@@ -394,6 +405,7 @@ impl PlanResolver<'_> {
                 state,
                 target_schema,
                 source_schema,
+                self.config.case_sensitive,
             );
             let resolved = self
                 .resolve_expression(disambiguated, merge_schema, state)
diff --git a/crates/sail-plan/src/resolver/command/explain.rs b/crates/sail-plan/src/resolver/command/explain.rs
index 544c153e1..25dcff046 100644
--- a/crates/sail-plan/src/resolver/command/explain.rs
+++ b/crates/sail-plan/src/resolver/command/explain.rs
@@ -1,12 +1,12 @@
 use async_recursion::async_recursion;
 use datafusion_common::ScalarValue;
-use datafusion_expr::{col, lit, LogicalPlan, LogicalPlanBuilder};
+use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, col, lit};
 use sail_common::spec;
 
 use crate::error::PlanResult;
-use crate::explain::{explain_string_from_logical_plan, ExplainOptions};
-use crate::resolver::state::PlanResolverState;
+use crate::explain::{ExplainOptions, explain_string_from_logical_plan};
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     #[async_recursion]
diff --git a/crates/sail-plan/src/resolver/command/function.rs b/crates/sail-plan/src/resolver/command/function.rs
index 43d8666e4..d7c8b29cf 100644
--- a/crates/sail-plan/src/resolver/command/function.rs
+++ b/crates/sail-plan/src/resolver/command/function.rs
@@ -6,8 +6,8 @@ use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_python_udf::udf::pyspark_unresolved_udf::PySparkUnresolvedUDF;
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) fn resolve_catalog_register_function(
diff --git a/crates/sail-plan/src/resolver/command/insert.rs b/crates/sail-plan/src/resolver/command/insert.rs
index ed8db1db8..bfee9450f 100644
--- a/crates/sail-plan/src/resolver/command/insert.rs
+++ b/crates/sail-plan/src/resolver/command/insert.rs
@@ -6,11 +6,11 @@ use sail_common_datafusion::catalog::{TableColumnStatus, TableKind, TemporaryVie
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::command::write::{
     TableInfo, WriteColumnMatch, WriteMode, WritePlanBuilder, WriteTarget,
 };
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_command_insert_overwrite_directory(
diff --git a/crates/sail-plan/src/resolver/command/merge.rs b/crates/sail-plan/src/resolver/command/merge.rs
index 2cdfa9fae..5e324aaba 100644
--- a/crates/sail-plan/src/resolver/command/merge.rs
+++ b/crates/sail-plan/src/resolver/command/merge.rs
@@ -1,13 +1,15 @@
+use std::collections::HashSet;
 use std::sync::Arc;
 
-use datafusion_common::{JoinType, TableReference};
-use datafusion_expr::utils::{expr_to_columns, split_conjunction};
-use datafusion_expr::{build_join_schema, Expr, LogicalPlan, SubqueryAlias};
+use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
+use datafusion_common::{DFSchema, JoinType, TableReference};
+use datafusion_expr::utils::{expr_to_columns, find_aggregate_exprs, split_conjunction};
+use datafusion_expr::{Expr, LogicalPlan, SubqueryAlias, build_join_schema};
 use sail_catalog::manager::CatalogManager;
 use sail_common::spec;
 use sail_common_datafusion::catalog::{LakehouseOperation, TableKind};
 use sail_common_datafusion::column_features::ColumnFeatures;
-use sail_common_datafusion::datasource::{MergeInfo, OptionLayer, TableFormatRegistry};
+use sail_common_datafusion::datasource::{MergeInfo, OptionLayer, SourceInfo, TableFormatRegistry};
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::logical_expr::ExprWithSource;
 use sail_logical_plan::merge::{
@@ -17,8 +19,8 @@ use sail_logical_plan::merge::{
 };
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 // When we receive an unqualified attribute with `plan_id=None`, we need:
 // - if it exists only in target -> treat as target
@@ -98,10 +100,12 @@ impl PlanResolver<'_> {
             state,
             target_schema,
             source_schema,
+            self.config.case_sensitive,
         );
         let on_condition = self
             .resolve_expression(on_condition_expr, &merge_schema, state)
             .await?;
+        validate_merge_condition(&on_condition)?;
         let (join_key_pairs, residual_predicates, target_only_predicates) =
             analyze_merge_join(&on_condition, &merge_schema, target_schema.clone());
 
@@ -114,6 +118,14 @@ impl PlanResolver<'_> {
                 state,
             )
             .await?;
+        self.validate_merge_star_columns(
+            &matched_clauses,
+            ¬_matched_by_target,
+            target_schema,
+            &resolved_target_field_names,
+            &resolved_source_field_names,
+            with_schema_evolution,
+        )?;
 
         let generated_column_exprs = self
             .resolve_delta_merge_generated_column_exprs(
@@ -123,6 +135,10 @@ impl PlanResolver<'_> {
                 state,
             )
             .await?;
+        // TODO: Resolve explicit DEFAULT values against their MERGE target columns.
+        let default_column_exprs = self
+            .resolve_merge_default_column_exprs(target_schema, &resolved_target_field_names, state)
+            .await?;
         let check_constraint_exprs = self
             .resolve_delta_merge_check_constraints(
                 &target_metadata.format,
@@ -138,10 +154,12 @@ impl PlanResolver<'_> {
             source_alias: source_alias_string,
             target: target_metadata,
             with_schema_evolution,
+            case_sensitive: self.config.case_sensitive,
             resolved_target_schema: target_schema.clone(),
             resolved_source_schema: source_schema.clone(),
             resolved_target_field_names,
             resolved_source_field_names,
+            source_column_aliases: vec![],
             on_condition: ExprWithSource::new(on_condition, on_condition_source),
             matched_clauses,
             not_matched_by_source_clauses: not_matched_by_source,
@@ -150,6 +168,7 @@ impl PlanResolver<'_> {
             residual_predicates,
             target_only_predicates,
             generated_column_exprs,
+            default_column_exprs,
             check_constraint_exprs,
         };
 
@@ -386,6 +405,11 @@ impl PlanResolver<'_> {
                         state,
                     )
                     .await?;
+                if columns.len() != values.len() {
+                    return Err(PlanError::invalid(
+                        "MERGE INSERT column and value counts do not match",
+                    ));
+                }
                 MergeNotMatchedByTargetAction::InsertColumns { columns, values }
             }
         };
@@ -401,15 +425,23 @@ impl PlanResolver<'_> {
         state: &mut PlanResolverState,
     ) -> PlanResult> {
         let mut out = Vec::with_capacity(assignments.len());
+        let mut assigned_columns = HashSet::with_capacity(assignments.len());
         for (column, value) in assignments {
             let resolved_column = self
                 .resolve_merge_column(column, target_schema.clone(), state)
                 .await?;
+            let assignment_key = self.merge_name_key(&resolved_column);
+            if !assigned_columns.insert(assignment_key) {
+                return Err(PlanError::invalid(format!(
+                    "Multiple assignments for MERGE target column `{resolved_column}`"
+                )));
+            }
             let value = merge_disambiguate_unqualified_plan_ids(
                 value,
                 state,
                 &target_schema,
                 source_schema,
+                self.config.case_sensitive,
             );
             let resolved_value = self.resolve_expression(value, merge_schema, state).await?;
             out.push(MergeAssignment {
@@ -427,11 +459,18 @@ impl PlanResolver<'_> {
         state: &mut PlanResolverState,
     ) -> PlanResult> {
         let mut out = Vec::with_capacity(columns.len());
+        let mut assigned_columns = HashSet::with_capacity(columns.len());
         for column in columns {
-            out.push(
-                self.resolve_merge_column(column, target_schema.clone(), state)
-                    .await?,
-            );
+            let resolved_column = self
+                .resolve_merge_column(column, target_schema.clone(), state)
+                .await?;
+            let assignment_key = self.merge_name_key(&resolved_column);
+            if !assigned_columns.insert(assignment_key) {
+                return Err(PlanError::invalid(format!(
+                    "Multiple assignments for MERGE target column `{resolved_column}`"
+                )));
+            }
+            out.push(resolved_column);
         }
         Ok(out)
     }
@@ -446,8 +485,13 @@ impl PlanResolver<'_> {
     ) -> PlanResult> {
         let mut out = Vec::with_capacity(values.len());
         for value in values {
-            let value =
-                merge_disambiguate_unqualified_plan_ids(value, state, target_schema, source_schema);
+            let value = merge_disambiguate_unqualified_plan_ids(
+                value,
+                state,
+                target_schema,
+                source_schema,
+                self.config.case_sensitive,
+            );
             out.push(self.resolve_expression(value, merge_schema, state).await?);
         }
         Ok(out)
@@ -482,25 +526,157 @@ impl PlanResolver<'_> {
         state: &mut PlanResolverState,
     ) -> PlanResult> {
         match expression {
-            Some(expr) => Ok(Some(ExprWithSource::new(
-                self.resolve_expression(
-                    merge_disambiguate_unqualified_plan_ids(
-                        expr.expr,
+            Some(expr) => {
+                let condition = self
+                    .resolve_expression(
+                        merge_disambiguate_unqualified_plan_ids(
+                            expr.expr,
+                            state,
+                            target_schema,
+                            source_schema,
+                            self.config.case_sensitive,
+                        ),
+                        schema,
                         state,
-                        target_schema,
-                        source_schema,
-                    ),
-                    schema,
-                    state,
-                )
-                .await?,
-                expr.source,
-            ))),
+                    )
+                    .await?;
+                validate_merge_condition(&condition)?;
+                Ok(Some(ExprWithSource::new(condition, expr.source)))
+            }
             None => Ok(None),
         }
     }
 
+    fn merge_name_key(&self, name: &str) -> String {
+        if self.config.case_sensitive {
+            name.to_string()
+        } else {
+            name.to_ascii_lowercase()
+        }
+    }
+
+    fn merge_names_equal(&self, left: &str, right: &str) -> bool {
+        if self.config.case_sensitive {
+            left == right
+        } else {
+            left.eq_ignore_ascii_case(right)
+        }
+    }
+
+    fn validate_merge_star_columns(
+        &self,
+        matched_clauses: &[MergeMatchedClause],
+        not_matched_by_target_clauses: &[MergeNotMatchedByTargetClause],
+        target_schema: &datafusion_common::DFSchemaRef,
+        target_names: &[String],
+        source_names: &[String],
+        with_schema_evolution: bool,
+    ) -> PlanResult<()> {
+        if with_schema_evolution
+            || (!matched_clauses
+                .iter()
+                .any(|clause| matches!(clause.action, MergeMatchedAction::UpdateAll))
+                && !not_matched_by_target_clauses.iter().any(|clause| {
+                    matches!(clause.action, MergeNotMatchedByTargetAction::InsertAll)
+                }))
+        {
+            return Ok(());
+        }
+
+        for (field, target_name) in target_schema.fields().iter().zip(target_names) {
+            if ColumnFeatures::from_field(field)
+                .generation_expression()
+                .is_some()
+            {
+                continue;
+            }
+            if !source_names
+                .iter()
+                .any(|source_name| self.merge_names_equal(source_name, target_name))
+            {
+                return Err(PlanError::invalid(format!(
+                    "Cannot resolve source column `{target_name}` for MERGE * action without schema evolution"
+                )));
+            }
+        }
+        Ok(())
+    }
+
+    async fn resolve_merge_default_column_exprs(
+        &self,
+        target_schema: &datafusion_common::DFSchemaRef,
+        target_names: &[String],
+        state: &mut PlanResolverState,
+    ) -> PlanResult> {
+        let empty_schema = Arc::new(DFSchema::empty());
+        let mut defaults = Vec::new();
+        for (field, target_name) in target_schema.fields().iter().zip(target_names) {
+            let Some(default) = ColumnFeatures::from_field(field).current_default() else {
+                continue;
+            };
+            let ast_expr =
+                sail_sql_analyzer::parser::parse_expression(&default).map_err(|error| {
+                    PlanError::invalid(format!(
+                        "failed to parse default expression `{default}`: {error}"
+                    ))
+                })?;
+            let spec_expr =
+                sail_sql_analyzer::expression::from_ast_expression(ast_expr).map_err(|error| {
+                    PlanError::invalid(format!(
+                        "failed to analyze default expression `{default}`: {error}"
+                    ))
+                })?;
+            let spec_expr = if matches!(spec_expr, spec::Expr::UnresolvedAttribute { .. }) {
+                spec::Expr::Literal(spec::Literal::Utf8 {
+                    value: Some(default),
+                })
+            } else {
+                spec_expr
+            };
+            let resolved = self
+                .resolve_expression(spec_expr, &empty_schema, state)
+                .await?;
+            defaults.push((target_name.clone(), resolved));
+        }
+        Ok(defaults)
+    }
+
     async fn get_merge_target_info(&self, table: &spec::ObjectName) -> PlanResult {
+        // Handle path-based table access like `delta.`/path/to/table``
+        // where the first part is a registered table format name.
+        if let [format, path] = table.parts() {
+            let format = format.as_ref().to_ascii_lowercase();
+            let registry = self.ctx.extension::()?;
+            if let Ok(table_format) = registry.get(&format) {
+                let location = path.as_ref().to_string();
+                let metadata = table_format
+                    .infer_metadata(
+                        &self.ctx.state(),
+                        SourceInfo {
+                            paths: vec![location.clone()],
+                            lakehouse_table: None,
+                            schema: None,
+                            constraints: Default::default(),
+                            partition_by: vec![],
+                            bucket_by: None,
+                            sort_order: vec![],
+                            options: vec![],
+                            read_case_sensitive: self.config.case_sensitive,
+                        },
+                    )
+                    .await?;
+                return Ok(MergeTargetInfo {
+                    table_name: table.clone().into(),
+                    format,
+                    location,
+                    partition_by: vec![],
+                    options: vec![OptionLayer::TablePropertyList {
+                        items: metadata.properties,
+                    }],
+                    lakehouse_table: None,
+                });
+            }
+        }
         let catalog_manager = self.ctx.extension::()?;
         let status = catalog_manager
             .get_table_or_view(table.parts())
@@ -542,24 +718,68 @@ impl PlanResolver<'_> {
     }
 }
 
+fn validate_merge_condition(condition: &Expr) -> PlanResult<()> {
+    if condition.is_volatile() {
+        return Err(PlanError::AnalysisError(
+            "Non-deterministic expressions are not allowed in MERGE conditions".to_string(),
+        ));
+    }
+
+    let mut contains_subquery = false;
+    condition.apply(|expr| {
+        if matches!(
+            expr,
+            Expr::Exists(_)
+                | Expr::InSubquery(_)
+                | Expr::SetComparison(_)
+                | Expr::ScalarSubquery(_)
+        ) {
+            contains_subquery = true;
+            Ok(TreeNodeRecursion::Stop)
+        } else {
+            Ok(TreeNodeRecursion::Continue)
+        }
+    })?;
+    if contains_subquery {
+        return Err(PlanError::AnalysisError(
+            "Subqueries are not allowed in MERGE conditions".to_string(),
+        ));
+    }
+
+    if !find_aggregate_exprs([condition]).is_empty() {
+        return Err(PlanError::AnalysisError(
+            "Aggregate expressions are not allowed in MERGE conditions".to_string(),
+        ));
+    }
+
+    Ok(())
+}
+
 fn merge_schema_has_column_name(
     schema: &datafusion_common::DFSchemaRef,
     state: &PlanResolverState,
     name: &str,
+    case_sensitive: bool,
 ) -> bool {
     schema.iter().any(|(_qualifier, field)| {
-        state
-            .get_field_info(field.name())
-            .is_ok_and(|info| !info.is_hidden() && info.name().eq_ignore_ascii_case(name))
+        state.get_field_info(field.name()).is_ok_and(|info| {
+            !info.is_hidden()
+                && if case_sensitive {
+                    info.name() == name
+                } else {
+                    info.name().eq_ignore_ascii_case(name)
+                }
+        })
     })
 }
 
-/// Disambiguate column references in a generation expression for MERGE INSERT/UPDATE context.
+/// Disambiguate unqualified MERGE references against the target and source schemas.
 pub(super) fn merge_disambiguate_unqualified_plan_ids(
     expr: spec::Expr,
     state: &PlanResolverState,
     target_schema: &datafusion_common::DFSchemaRef,
     source_schema: &datafusion_common::DFSchemaRef,
+    case_sensitive: bool,
 ) -> spec::Expr {
     use spec::Expr;
 
@@ -573,8 +793,10 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
             // like `t.id` should be resolved using the qualifier.
             if let [part] = name.parts() {
                 let col = part.as_ref();
-                let in_target = merge_schema_has_column_name(target_schema, state, col);
-                let in_source = merge_schema_has_column_name(source_schema, state, col);
+                let in_target =
+                    merge_schema_has_column_name(target_schema, state, col, case_sensitive);
+                let in_source =
+                    merge_schema_has_column_name(source_schema, state, col, case_sensitive);
                 let plan_id = match (in_target, in_source) {
                     (true, false) => Some(MERGE_TARGET_DEFAULT_PLAN_ID),
                     (false, true) => Some(MERGE_SOURCE_DEFAULT_PLAN_ID),
@@ -602,7 +824,13 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 .arguments
                 .into_iter()
                 .map(|e| {
-                    merge_disambiguate_unqualified_plan_ids(e, state, target_schema, source_schema)
+                    merge_disambiguate_unqualified_plan_ids(
+                        e,
+                        state,
+                        target_schema,
+                        source_schema,
+                        case_sensitive,
+                    )
                 })
                 .collect();
             Expr::UnresolvedFunction(f)
@@ -627,6 +855,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             name,
             metadata,
@@ -642,6 +871,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             cast_to_type,
             rename,
@@ -661,6 +891,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             ..sort
         }),
@@ -673,6 +904,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             arguments,
         },
@@ -685,6 +917,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             window,
         },
@@ -694,12 +927,14 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             extraction: Box::new(merge_disambiguate_unqualified_plan_ids(
                 *extraction,
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
         Expr::UpdateFields {
@@ -712,6 +947,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             field_name,
             value_expression: value_expression.map(|v| {
@@ -720,6 +956,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                     state,
                     target_schema,
                     source_schema,
+                    case_sensitive,
                 ))
             }),
         },
@@ -733,7 +970,13 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
             arguments: arguments
                 .into_iter()
                 .map(|e| {
-                    merge_disambiguate_unqualified_plan_ids(e, state, target_schema, source_schema)
+                    merge_disambiguate_unqualified_plan_ids(
+                        e,
+                        state,
+                        target_schema,
+                        source_schema,
+                        case_sensitive,
+                    )
                 })
                 .collect(),
         },
@@ -742,7 +985,13 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
             exprs
                 .into_iter()
                 .map(|e| {
-                    merge_disambiguate_unqualified_plan_ids(e, state, target_schema, source_schema)
+                    merge_disambiguate_unqualified_plan_ids(
+                        e,
+                        state,
+                        target_schema,
+                        source_schema,
+                        case_sensitive,
+                    )
                 })
                 .collect(),
         ),
@@ -750,7 +999,13 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
             exprs
                 .into_iter()
                 .map(|e| {
-                    merge_disambiguate_unqualified_plan_ids(e, state, target_schema, source_schema)
+                    merge_disambiguate_unqualified_plan_ids(
+                        e,
+                        state,
+                        target_schema,
+                        source_schema,
+                        case_sensitive,
+                    )
                 })
                 .collect(),
         ),
@@ -764,6 +1019,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                                 state,
                                 target_schema,
                                 source_schema,
+                                case_sensitive,
                             )
                         })
                         .collect()
@@ -780,6 +1036,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             subquery,
             negated,
@@ -796,11 +1053,18 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             list: list
                 .into_iter()
                 .map(|e| {
-                    merge_disambiguate_unqualified_plan_ids(e, state, target_schema, source_schema)
+                    merge_disambiguate_unqualified_plan_ids(
+                        e,
+                        state,
+                        target_schema,
+                        source_schema,
+                        case_sensitive,
+                    )
                 })
                 .collect(),
             negated,
@@ -810,46 +1074,59 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
         Expr::IsNotFalse(e) => Expr::IsNotFalse(Box::new(merge_disambiguate_unqualified_plan_ids(
             *e,
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
         Expr::IsTrue(e) => Expr::IsTrue(Box::new(merge_disambiguate_unqualified_plan_ids(
             *e,
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
         Expr::IsNotTrue(e) => Expr::IsNotTrue(Box::new(merge_disambiguate_unqualified_plan_ids(
             *e,
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
         Expr::IsNull(e) => Expr::IsNull(Box::new(merge_disambiguate_unqualified_plan_ids(
             *e,
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
         Expr::IsNotNull(e) => Expr::IsNotNull(Box::new(merge_disambiguate_unqualified_plan_ids(
             *e,
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
         Expr::IsUnknown(e) => Expr::IsUnknown(Box::new(merge_disambiguate_unqualified_plan_ids(
             *e,
             state,
             target_schema,
             source_schema,
+            case_sensitive,
         ))),
-        Expr::IsNotUnknown(e) => Expr::IsNotUnknown(Box::new(
-            merge_disambiguate_unqualified_plan_ids(*e, state, target_schema, source_schema),
-        )),
+        Expr::IsNotUnknown(e) => {
+            Expr::IsNotUnknown(Box::new(merge_disambiguate_unqualified_plan_ids(
+                *e,
+                state,
+                target_schema,
+                source_schema,
+                case_sensitive,
+            )))
+        }
         Expr::Between {
             expr,
             negated,
@@ -861,6 +1138,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             negated,
             low: Box::new(merge_disambiguate_unqualified_plan_ids(
@@ -868,12 +1146,14 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             high: Box::new(merge_disambiguate_unqualified_plan_ids(
                 *high,
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
         Expr::IsDistinctFrom { left, right } => Expr::IsDistinctFrom {
@@ -882,12 +1162,14 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             right: Box::new(merge_disambiguate_unqualified_plan_ids(
                 *right,
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
         Expr::IsNotDistinctFrom { left, right } => Expr::IsNotDistinctFrom {
@@ -896,12 +1178,14 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             right: Box::new(merge_disambiguate_unqualified_plan_ids(
                 *right,
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
         Expr::SimilarTo {
@@ -916,12 +1200,14 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             pattern: Box::new(merge_disambiguate_unqualified_plan_ids(
                 *pattern,
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
             negated,
             escape_char,
@@ -933,6 +1219,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
         Expr::UnresolvedDate { .. } => expr,
@@ -943,6 +1230,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
         Expr::Subquery {
@@ -961,6 +1249,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                         state,
                         target_schema,
                         source_schema,
+                        case_sensitive,
                     )
                 })
                 .collect(),
@@ -974,6 +1263,7 @@ pub(super) fn merge_disambiguate_unqualified_plan_ids(
                 state,
                 target_schema,
                 source_schema,
+                case_sensitive,
             )),
         },
     }
@@ -1001,14 +1291,17 @@ fn classify_expr_domain(
         return ExprColumnDomain::Mixed;
     }
     for col in cols.into_iter() {
-        if let Ok(idx) = merge_schema.index_of_column(&col) {
-            if idx < target_len {
-                seen_target = true;
-            } else {
-                seen_source = true;
+        match merge_schema.index_of_column(&col) {
+            Ok(idx) => {
+                if idx < target_len {
+                    seen_target = true;
+                } else {
+                    seen_source = true;
+                }
+            }
+            _ => {
+                return ExprColumnDomain::Mixed;
             }
-        } else {
-            return ExprColumnDomain::Mixed;
         }
     }
     match (seen_target, seen_source) {
@@ -1033,21 +1326,21 @@ fn analyze_merge_join(
     let target_len = target_schema.fields().len();
 
     for predicate in split_conjunction(on_condition) {
-        if let Expr::BinaryExpr(be) = predicate {
-            if be.op == datafusion_expr::Operator::Eq {
-                let left_domain = classify_expr_domain(&be.left, merge_schema, target_len);
-                let right_domain = classify_expr_domain(&be.right, merge_schema, target_len);
-                match (left_domain, right_domain) {
-                    (ExprColumnDomain::TargetOnly, ExprColumnDomain::SourceOnly) => {
-                        join_key_pairs.push(((*be.left).clone(), (*be.right).clone()));
-                        continue;
-                    }
-                    (ExprColumnDomain::SourceOnly, ExprColumnDomain::TargetOnly) => {
-                        join_key_pairs.push(((*be.right).clone(), (*be.left).clone()));
-                        continue;
-                    }
-                    _ => {}
+        if let Expr::BinaryExpr(be) = predicate
+            && be.op == datafusion_expr::Operator::Eq
+        {
+            let left_domain = classify_expr_domain(&be.left, merge_schema, target_len);
+            let right_domain = classify_expr_domain(&be.right, merge_schema, target_len);
+            match (left_domain, right_domain) {
+                (ExprColumnDomain::TargetOnly, ExprColumnDomain::SourceOnly) => {
+                    join_key_pairs.push(((*be.left).clone(), (*be.right).clone()));
+                    continue;
+                }
+                (ExprColumnDomain::SourceOnly, ExprColumnDomain::TargetOnly) => {
+                    join_key_pairs.push(((*be.right).clone(), (*be.left).clone()));
+                    continue;
                 }
+                _ => {}
             }
         }
 
diff --git a/crates/sail-plan/src/resolver/command/mod.rs b/crates/sail-plan/src/resolver/command/mod.rs
index 2bd664756..5111c21b8 100644
--- a/crates/sail-plan/src/resolver/command/mod.rs
+++ b/crates/sail-plan/src/resolver/command/mod.rs
@@ -1,5 +1,6 @@
 use std::sync::Arc;
 
+use async_recursion::async_recursion;
 use datafusion_expr::{Extension, LogicalPlan};
 use sail_catalog::command::CatalogCommand;
 use sail_catalog::provider::{DropDatabaseOptions, DropTableOptions};
@@ -7,10 +8,12 @@ use sail_common::spec;
 
 use crate::catalog::CatalogCommandNode;
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
+use crate::function::list_built_in_function_statuses;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 mod catalog;
+mod checkpoint;
 mod delete;
 mod delta;
 mod explain;
@@ -19,6 +22,7 @@ mod insert;
 mod merge;
 mod show;
 mod variable;
+mod with_relations;
 mod write;
 mod write_stream;
 mod write_v1;
@@ -26,6 +30,7 @@ mod write_v2;
 
 impl PlanResolver<'_> {
     /// Resolves a command plan into a logical plan.
+    #[async_recursion]
     pub(super) async fn resolve_command_plan(
         &self,
         plan: spec::CommandPlan,
@@ -36,6 +41,10 @@ impl PlanResolver<'_> {
         match plan.node {
             CommandNode::ShowString(show) => self.resolve_command_show_string(show, state).await,
             CommandNode::HtmlString(html) => self.resolve_command_html_string(html, state).await,
+            CommandNode::WithRelations { root, references } => {
+                self.resolve_command_with_relations(*root, references, state)
+                    .await
+            }
             CommandNode::CurrentDatabase => {
                 self.resolve_catalog_command(CatalogCommand::CurrentDatabase)
             }
@@ -62,6 +71,18 @@ impl PlanResolver<'_> {
                     pattern,
                 })
             }
+            CommandNode::ShowFunctions {
+                database,
+                pattern,
+                show_user_functions,
+                show_system_functions,
+            } => self.resolve_catalog_command(CatalogCommand::ShowFunctions {
+                database: database.map(|x| x.into()).unwrap_or_default(),
+                pattern,
+                system_functions: list_built_in_function_statuses(),
+                show_user_functions,
+                show_system_functions,
+            }),
             CommandNode::ListTables { database, pattern } => {
                 self.resolve_catalog_command(CatalogCommand::ListTables {
                     database: database.map(|x| x.into()).unwrap_or_default(),
@@ -78,6 +99,7 @@ impl PlanResolver<'_> {
                 self.resolve_catalog_command(CatalogCommand::ListFunctions {
                     database: database.map(|x| x.into()).unwrap_or_default(),
                     pattern,
+                    system_functions: list_built_in_function_statuses(),
                 })
             }
             CommandNode::ListColumns { table } => {
@@ -209,6 +231,10 @@ impl PlanResolver<'_> {
                 self.resolve_catalog_create_temporary_view(view, is_global, definition, state)
                     .await
             }
+            CommandNode::RemoteCheckpoint { relation_id, input } => {
+                self.resolve_command_remote_checkpoint(relation_id, *input, state)
+                    .await
+            }
             CommandNode::Write(write) => self.resolve_command_write(write, state).await,
             CommandNode::WriteTo(write_to) => self.resolve_command_write_to(write_to, state).await,
             CommandNode::WriteStream(write_stream) => {
@@ -283,8 +309,12 @@ impl PlanResolver<'_> {
             CommandNode::AnalyzeTable { .. } => Err(PlanError::todo("CommandNode::AnalyzeTable")),
             CommandNode::AnalyzeTables { .. } => Err(PlanError::todo("CommandNode::AnalyzeTables")),
             CommandNode::DescribeQuery { .. } => Err(PlanError::todo("CommandNode::DescribeQuery")),
-            CommandNode::DescribeFunction { .. } => {
-                Err(PlanError::todo("CommandNode::DescribeFunction"))
+            CommandNode::DescribeFunction { function, extended } => {
+                self.resolve_catalog_command(CatalogCommand::DescribeFunction {
+                    function: function.into(),
+                    extended,
+                    system_functions: list_built_in_function_statuses(),
+                })
             }
             CommandNode::DescribeCatalog { .. } => {
                 Err(PlanError::todo("CommandNode::DescribeCatalog"))
diff --git a/crates/sail-plan/src/resolver/command/show.rs b/crates/sail-plan/src/resolver/command/show.rs
index cf91a28b1..b21fa503d 100644
--- a/crates/sail-plan/src/resolver/command/show.rs
+++ b/crates/sail-plan/src/resolver/command/show.rs
@@ -1,12 +1,12 @@
 use std::sync::Arc;
 
-use datafusion_expr::{lit, Extension, Limit, LogicalPlan};
+use datafusion_expr::{Extension, Limit, LogicalPlan, lit};
 use sail_common::spec;
 use sail_logical_plan::show_string::{ShowStringFormat, ShowStringNode, ShowStringStyle};
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_command_show_string(
diff --git a/crates/sail-plan/src/resolver/command/with_relations.rs b/crates/sail-plan/src/resolver/command/with_relations.rs
new file mode 100644
index 000000000..1e482ee50
--- /dev/null
+++ b/crates/sail-plan/src/resolver/command/with_relations.rs
@@ -0,0 +1,65 @@
+use std::sync::Arc;
+
+use async_recursion::async_recursion;
+use datafusion_expr::{LogicalPlan, SubqueryAlias};
+use sail_common::spec;
+
+use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
+
+impl PlanResolver<'_> {
+    /// Resolves a command `WithRelations` node by storing references in state, then resolving
+    /// the root command. This is the command analogue of `resolve_query_with_relations`.
+    ///
+    /// This is used when a command (e.g., `MERGE INTO`) references DataFrames passed as named
+    /// arguments via `spark.sql("MERGE INTO t USING {df}", df=dataframe)`. PySpark wraps the
+    /// DataFrame in a `SubqueryAlias` within a `WithRelations` node whose root is the command.
+    #[async_recursion]
+    pub(super) async fn resolve_command_with_relations(
+        &self,
+        root: spec::CommandPlan,
+        references: Vec,
+        state: &mut PlanResolverState,
+    ) -> PlanResult {
+        let mut scope = state.enter_with_relations_scope();
+        let state = scope.state();
+        // Enter a CTE scope so that DataFrame aliases registered below do not leak
+        // into the outer command scope.
+        let mut scope = state.enter_cte_scope();
+        let state = scope.state();
+        for ref_plan in references {
+            // If the reference is a SubqueryAlias, register it as a named table so that
+            // SQL commands can reference the DataFrame by its alias name.
+            // This enables the `spark.sql("MERGE INTO t USING {df}", df=dataframe)` pattern where
+            // PySpark replaces `{df}` with a generated temp view name and wraps the DataFrame
+            // in a SubqueryAlias within a WithRelations node.
+            if let spec::QueryNode::SubqueryAlias {
+                ref input,
+                ref alias,
+                ref qualifier,
+            } = ref_plan.node
+            {
+                let resolved = self.resolve_query_plan((**input).clone(), state).await?;
+                let table_ref = self.resolve_table_reference(
+                    &spec::ObjectName::from(qualifier.clone()).child(alias.clone()),
+                )?;
+                let aliased = LogicalPlan::SubqueryAlias(SubqueryAlias::try_new(
+                    Arc::new(resolved),
+                    table_ref.clone(),
+                )?);
+                state.insert_cte(table_ref, aliased);
+            }
+            let plan_id = ref_plan
+                .plan_id
+                .ok_or_else(|| PlanError::invalid("subquery reference missing plan_id"))?;
+            if state.insert_subquery_reference(plan_id, ref_plan).is_some() {
+                return Err(PlanError::invalid(format!(
+                    "duplicate subquery reference for plan_id {}",
+                    plan_id
+                )));
+            }
+        }
+        self.resolve_command_plan(root, scope.state()).await
+    }
+}
diff --git a/crates/sail-plan/src/resolver/command/write.rs b/crates/sail-plan/src/resolver/command/write.rs
index 27f4bb3b5..4abb3943d 100644
--- a/crates/sail-plan/src/resolver/command/write.rs
+++ b/crates/sail-plan/src/resolver/command/write.rs
@@ -8,8 +8,8 @@ use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
 use datafusion_common::{Column, DFSchema, ScalarValue};
 use datafusion_expr::expr::{self, FieldMetadata, Sort, WindowFunctionParams};
 use datafusion_expr::{
-    col, lit, when, BinaryExpr, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder,
-    Operator, Projection, ScalarUDF, WindowFrame, WindowFunctionDefinition,
+    BinaryExpr, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, Operator,
+    Projection, ScalarUDF, WindowFrame, WindowFunctionDefinition, col, lit, when,
 };
 use sail_catalog::command::CatalogCommand;
 use sail_catalog::error::CatalogError;
@@ -28,8 +28,8 @@ use sail_common_datafusion::column_features::{
     ColumnFeatures, ColumnFeaturesBuilder, SAIL_WRITE_TARGET_NULLABLE_METADATA_KEY,
 };
 use sail_common_datafusion::datasource::{
-    find_path_in_options, BucketBy, OptionLayer, SinkInfo, SinkMode, SourceInfo,
-    TableFormatRegistry,
+    BucketBy, OptionLayer, SinkInfo, SinkMode, SourceInfo, TableFormatRegistry,
+    find_path_in_options,
 };
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::logical_expr::ExprWithSource;
@@ -43,8 +43,8 @@ use sail_logical_plan::barrier::BarrierNode;
 
 use super::delta::parse_delta_generation_expr;
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 /// The write modes for all targets.
 ///
@@ -463,7 +463,7 @@ impl PlanResolver<'_> {
                         .iter()
                         .map(|part| part.as_ref().to_string())
                         .collect::>();
-                    let create_options = CreateTableOptions {
+                    let mut create_options = CreateTableOptions {
                         columns,
                         comment: None,
                         constraints: vec![],
@@ -488,6 +488,14 @@ impl PlanResolver<'_> {
                             },
                         )
                         .await?;
+                    if let Some(location) = create_plan.table.status.location.clone()
+                        && create_options.location.as_deref() != Some(location.as_str())
+                    {
+                        create_options.location = Some(location.clone());
+                        sink_info.options.push(OptionLayer::OptionList {
+                            items: vec![("path".to_string(), location)],
+                        });
+                    }
                     sink_info.lakehouse_table = Some(
                         create_plan
                             .table
diff --git a/crates/sail-plan/src/resolver/command/write_stream.rs b/crates/sail-plan/src/resolver/command/write_stream.rs
index afb418976..f9acf471e 100644
--- a/crates/sail-plan/src/resolver/command/write_stream.rs
+++ b/crates/sail-plan/src/resolver/command/write_stream.rs
@@ -3,9 +3,9 @@ use sail_catalog::provider::CatalogPartitionField;
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::command::write::{WriteColumnMatch, WriteMode, WritePlanBuilder, WriteTarget};
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     /// Resolves the write operation for the Spark streaming query API.
diff --git a/crates/sail-plan/src/resolver/command/write_v1.rs b/crates/sail-plan/src/resolver/command/write_v1.rs
index 3298784e5..c95ec70ed 100644
--- a/crates/sail-plan/src/resolver/command/write_v1.rs
+++ b/crates/sail-plan/src/resolver/command/write_v1.rs
@@ -2,9 +2,9 @@ use datafusion_expr::LogicalPlan;
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::command::write::{WriteColumnMatch, WriteMode, WritePlanBuilder, WriteTarget};
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     /// Resolves the write operation for the Spark DataFrameWriter v1 API.
diff --git a/crates/sail-plan/src/resolver/command/write_v2.rs b/crates/sail-plan/src/resolver/command/write_v2.rs
index 11f67f881..1c1d03c98 100644
--- a/crates/sail-plan/src/resolver/command/write_v2.rs
+++ b/crates/sail-plan/src/resolver/command/write_v2.rs
@@ -2,9 +2,9 @@ use datafusion_expr::LogicalPlan;
 use sail_common::spec;
 
 use crate::error::PlanResult;
+use crate::resolver::PlanResolver;
 use crate::resolver::command::write::{WriteColumnMatch, WriteMode, WritePlanBuilder, WriteTarget};
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     /// Resolves the write operation for the Spark DataFrameWriter v2 API.
diff --git a/crates/sail-plan/src/resolver/data_type.rs b/crates/sail-plan/src/resolver/data_type.rs
index d68fc634d..b62d050d6 100644
--- a/crates/sail-plan/src/resolver/data_type.rs
+++ b/crates/sail-plan/src/resolver/data_type.rs
@@ -2,7 +2,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use arrow_schema::extension::{
-    ExtensionType, EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY,
+    EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType,
 };
 use datafusion::arrow::datatypes as adt;
 use parquet_variant_compute::VariantType;
@@ -11,12 +11,13 @@ use sail_common::spec;
 use sail_common::spec::{
     SAIL_LIST_FIELD_NAME, SAIL_MAP_FIELD_NAME, SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME,
 };
+use sail_common_datafusion::variant::{VARIANT_VALUE_FIELD_NAME, variant_metadata_field};
 use serde_json::json;
 
 use crate::config::DefaultTimestampType;
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 /// Map SRID to CRS string for Arrow metadata.
 ///
@@ -282,10 +283,8 @@ impl PlanResolver<'_> {
                 // Variant layout using Binary for PySpark compatibility.
                 // parquet-variant uses BinaryView internally but we convert to Binary
                 let fields = adt::Fields::from(vec![
-                    adt::Field::new("value", adt::DataType::Binary, false),
-                    adt::Field::new("metadata", adt::DataType::Binary, false).with_metadata(
-                        HashMap::from([("variant".to_string(), "true".to_string())]),
-                    ),
+                    adt::Field::new(VARIANT_VALUE_FIELD_NAME, adt::DataType::Binary, false),
+                    variant_metadata_field(adt::DataType::Binary, false),
                 ]);
                 Ok(adt::DataType::Struct(fields))
             }
diff --git a/crates/sail-plan/src/resolver/expression/attribute.rs b/crates/sail-plan/src/resolver/expression/attribute.rs
index 17bc7a185..52d5e8456 100644
--- a/crates/sail-plan/src/resolver/expression/attribute.rs
+++ b/crates/sail-plan/src/resolver/expression/attribute.rs
@@ -3,15 +3,15 @@ use std::sync::Arc;
 use arrow::datatypes::{DataType, Field};
 use datafusion_common::{Column, DFSchemaRef, TableReference};
 use datafusion_expr::expr::{LambdaVariable, ScalarFunction};
-use datafusion_expr::{col, expr, lit, ScalarUDF};
+use datafusion_expr::{ScalarUDF, col, expr, lit};
 use datafusion_functions::core::get_field;
 use sail_common::spec;
 use sail_function::scalar::array_struct_field::ArrayStructField;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) fn resolve_expression_attribute(
@@ -29,27 +29,25 @@ impl PlanResolver<'_> {
         // bodies reference parameters as plain attributes, so the lambda scope stack
         // is consulted first. A `plan_id` indicates an explicit DataFrame column
         // reference, which never refers to a lambda parameter.
-        if plan_id.is_none() {
-            if let [first, rest @ ..] = name.parts() {
-                if let Some((declared, field)) = state
-                    .resolve_lambda_parameter(first.as_ref())
-                    .map(|(param, field)| (param.to_string(), field.cloned()))
-                {
-                    let display = rest
-                        .last()
-                        .map(|x| x.as_ref())
-                        .unwrap_or(declared.as_str())
-                        .to_string();
-                    let mut expr = expr::Expr::LambdaVariable(LambdaVariable::new(declared, field));
-                    for part in rest {
-                        expr = expr::Expr::ScalarFunction(ScalarFunction::new_udf(
-                            get_field(),
-                            vec![expr, lit(part.as_ref().to_string())],
-                        ));
-                    }
-                    return Ok(NamedExpr::new(vec![display], expr));
-                }
+        if plan_id.is_none()
+            && let [first, rest @ ..] = name.parts()
+            && let Some((declared, field)) = state
+                .resolve_lambda_parameter(first.as_ref())
+                .map(|(param, field)| (param.to_string(), field.cloned()))
+        {
+            let display = rest
+                .last()
+                .map(|x| x.as_ref())
+                .unwrap_or(declared.as_str())
+                .to_string();
+            let mut expr = expr::Expr::LambdaVariable(LambdaVariable::new(declared, field));
+            for part in rest {
+                expr = expr::Expr::ScalarFunction(ScalarFunction::new_udf(
+                    get_field(),
+                    vec![expr, lit(part.as_ref().to_string())],
+                ));
             }
+            return Ok(NamedExpr::new(vec![display], expr));
         }
         if let Some((name, expr)) =
             self.resolve_aggregate_field(&name, state.get_grouping_for_having())?
diff --git a/crates/sail-plan/src/resolver/expression/cast.rs b/crates/sail-plan/src/resolver/expression/cast.rs
index 8875f7c47..1f2e4ae41 100644
--- a/crates/sail-plan/src/resolver/expression/cast.rs
+++ b/crates/sail-plan/src/resolver/expression/cast.rs
@@ -3,9 +3,9 @@ use std::sync::Arc;
 
 use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit};
 use datafusion_common::{DFSchemaRef, ScalarValue};
-use datafusion_expr::{cast, expr, lit, try_cast, ExprSchemable, ScalarUDF};
-use sail_common::datetime::time_unit_to_multiplier;
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, lit, try_cast};
 use sail_common::spec;
+use sail_common::utils::datetime::time_unit_to_multiplier;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::plan::PlanService;
 use sail_common_datafusion::utils::items::ItemTaker;
@@ -22,9 +22,9 @@ use sail_function::scalar::variant::spark_variant_get::SparkVariantGet;
 use sail_function::scalar::variant::spark_variant_to_json::SparkVariantToJsonUdf;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_cast(
@@ -166,7 +166,9 @@ impl PlanResolver<'_> {
                 DataType::Timestamp(TimeUnit::Microsecond, tz),
                 is_try,
             ) => Arc::new(ScalarUDF::new_from_impl(SparkTimestamp::try_new(
-                tz, is_try,
+                tz,
+                self.config.ansi_mode,
+                is_try,
             )?))
             .call(vec![expr]),
             (_, DataType::Utf8, _) if override_string_cast => {
diff --git a/crates/sail-plan/src/resolver/expression/function.rs b/crates/sail-plan/src/resolver/expression/function.rs
index 68ccc0b53..32b0d37c9 100644
--- a/crates/sail-plan/src/resolver/expression/function.rs
+++ b/crates/sail-plan/src/resolver/expression/function.rs
@@ -1,7 +1,7 @@
 use datafusion_common::DFSchemaRef;
 use datafusion_expr::expr::ScalarFunction;
 use datafusion_expr::utils::{expand_qualified_wildcard, expand_wildcard};
-use datafusion_expr::{expr, EmptyRelation, Expr, LogicalPlan};
+use datafusion_expr::{EmptyRelation, Expr, LogicalPlan, expr};
 use sail_catalog::manager::CatalogManager;
 use sail_common::spec;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
@@ -15,11 +15,11 @@ use crate::function::common::{AggFunctionInput, FunctionContextInput, ScalarFunc
 use crate::function::{
     get_built_in_aggregate_function, get_built_in_function, is_higher_order_function,
 };
-use crate::resolver::expression::lambda::is_spec_lambda_argument;
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
+use crate::resolver::expression::lambda::is_spec_lambda_argument;
 use crate::resolver::function::PythonUdf;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_function(
@@ -73,10 +73,10 @@ impl PlanResolver<'_> {
 
         let canonical_function_name = function_name.to_ascii_lowercase();
         let catalog_manager = self.ctx.extension::()?;
-        if let Some(udf) = catalog_manager.get_function(&canonical_function_name)? {
-            if udf.inner().is::() {
-                state.config_mut().arrow_allow_large_var_types = true;
-            }
+        if let Some(udf) = catalog_manager.get_function(&canonical_function_name)?
+            && udf.inner().is::()
+        {
+            state.config_mut().arrow_allow_large_var_types = true;
         }
 
         // For functions that accept a date-part keyword as the first argument
@@ -113,111 +113,128 @@ impl PlanResolver<'_> {
 
         // FIXME: `is_user_defined_function` is always false,
         //   so we need to check UDFs before built-in functions.
-        let func = if let Some(udf) = catalog_manager.get_function(&canonical_function_name)? {
-            if ignore_nulls.is_some() || filter.is_some() || order_by.is_some() {
-                return Err(PlanError::invalid("invalid scalar function clause"));
-            }
-            if let Some(f) = udf.inner().downcast_ref::() {
-                if f.eval_type().is_table_function() {
-                    return Err(PlanError::AnalysisError(format!(
-                        "user-defined table function cannot be used as a scalar function: {function_name}"
-                    )));
+        let func = match catalog_manager.get_function(&canonical_function_name)? {
+            Some(udf) => {
+                if ignore_nulls.is_some() || filter.is_some() || order_by.is_some() {
+                    return Err(PlanError::invalid("invalid scalar function clause"));
+                }
+                if let Some(f) = udf.inner().downcast_ref::() {
+                    if f.eval_type().is_table_function() {
+                        return Err(PlanError::AnalysisError(format!(
+                            "user-defined table function cannot be used as a scalar function: {function_name}"
+                        )));
+                    }
+                    let output_type = f.output_type().cloned().ok_or_else(|| {
+                        PlanError::internal(format!(
+                            "unresolved UDF {function_name} has no scalar return type"
+                        ))
+                    })?;
+                    let function = PythonUdf {
+                        python_version: f.python_version().to_string(),
+                        eval_type: f.eval_type(),
+                        command: f.command().to_vec(),
+                        output_type,
+                    };
+                    self.resolve_python_udf_expr(
+                        function,
+                        &function_name,
+                        arguments,
+                        &argument_display_names,
+                        &kwarg_names, // pass kwargs from named_arguments
+                        schema,
+                        f.deterministic(),
+                        is_distinct,
+                        state,
+                    )?
+                } else {
+                    expr::Expr::ScalarFunction(ScalarFunction {
+                        func: std::sync::Arc::new(udf),
+                        args: arguments,
+                    })
                 }
-                let output_type = f.output_type().cloned().ok_or_else(|| {
-                    PlanError::internal(format!(
-                        "unresolved UDF {function_name} has no scalar return type"
-                    ))
-                })?;
-                let function = PythonUdf {
-                    python_version: f.python_version().to_string(),
-                    eval_type: f.eval_type(),
-                    command: f.command().to_vec(),
-                    output_type,
-                };
-                self.resolve_python_udf_expr(
-                    function,
-                    &function_name,
-                    arguments,
-                    &argument_display_names,
-                    &kwarg_names, // pass kwargs from named_arguments
-                    schema,
-                    f.deterministic(),
-                    is_distinct,
-                    state,
-                )?
-            } else {
-                expr::Expr::ScalarFunction(ScalarFunction {
-                    func: std::sync::Arc::new(udf),
-                    args: arguments,
-                })
             }
-        } else if let Ok(func) = get_built_in_function(&canonical_function_name) {
-            if ignore_nulls.is_some() || filter.is_some() || order_by.is_some() {
-                return Err(PlanError::invalid("invalid scalar function clause"));
+            _ => {
+                match get_built_in_function(&canonical_function_name) {
+                    Ok(func) => {
+                        if ignore_nulls.is_some() || filter.is_some() || order_by.is_some() {
+                            return Err(PlanError::invalid("invalid scalar function clause"));
+                        }
+                        let input = ScalarFunctionInput {
+                            arguments,
+                            function_context: FunctionContextInput {
+                                argument_display_names: &argument_display_names,
+                                plan_config: &self.config,
+                                session_context: self.ctx,
+                                schema,
+                            },
+                        };
+                        func(input)?
+                    }
+                    _ => {
+                        match get_built_in_aggregate_function(&canonical_function_name) {
+                            Ok(func) => {
+                                let filter = match filter {
+                                    Some(x) => Some(Box::new(
+                                        self.resolve_expression(*x, schema, state).await?,
+                                    )),
+                                    None => None,
+                                };
+                                let order_by = match order_by {
+                                    Some(x) => {
+                                        self.resolve_sort_orders(x, true, schema, state).await?
+                                    }
+                                    None => vec![],
+                                };
+                                // For DISTINCT aggregate functions with a wildcard argument (e.g., COUNT(DISTINCT *)),
+                                // expand the wildcard to visible column references here in the resolver where we have
+                                // access to `state` for hidden-column filtering. This ensures hidden columns (e.g.,
+                                // join keys) are excluded from the distinct count.
+                                #[expect(deprecated)]
+                                let arguments = if is_distinct
+                                    && matches!(
+                                        arguments.as_slice(),
+                                        [expr::Expr::Wildcard {
+                                            qualifier: None,
+                                            options: _
+                                        }]
+                                    ) {
+                                    schema
+                                        .columns()
+                                        .into_iter()
+                                        .filter(|c| {
+                                            state
+                                                .get_field_info(&c.name)
+                                                .is_ok_and(|info| !info.is_hidden())
+                                        })
+                                        .map(expr::Expr::Column)
+                                        .collect()
+                                } else {
+                                    arguments
+                                };
+                                let input = AggFunctionInput {
+                                    arguments,
+                                    distinct: is_distinct,
+                                    ignore_nulls,
+                                    filter,
+                                    order_by,
+                                    function_context: FunctionContextInput {
+                                        argument_display_names: &argument_display_names,
+                                        plan_config: &self.config,
+                                        session_context: self.ctx,
+                                        schema,
+                                    },
+                                };
+                                func(input)?
+                            }
+                            _ => {
+                                return Err(PlanError::unsupported(format!(
+                                    "unknown function: {function_name}",
+                                )));
+                            }
+                        }
+                    }
+                }
             }
-            let input = ScalarFunctionInput {
-                arguments,
-                function_context: FunctionContextInput {
-                    argument_display_names: &argument_display_names,
-                    plan_config: &self.config,
-                    session_context: self.ctx,
-                    schema,
-                },
-            };
-            func(input)?
-        } else if let Ok(func) = get_built_in_aggregate_function(&canonical_function_name) {
-            let filter = match filter {
-                Some(x) => Some(Box::new(self.resolve_expression(*x, schema, state).await?)),
-                None => None,
-            };
-            let order_by = match order_by {
-                Some(x) => self.resolve_sort_orders(x, true, schema, state).await?,
-                None => vec![],
-            };
-            // For DISTINCT aggregate functions with a wildcard argument (e.g., COUNT(DISTINCT *)),
-            // expand the wildcard to visible column references here in the resolver where we have
-            // access to `state` for hidden-column filtering. This ensures hidden columns (e.g.,
-            // join keys) are excluded from the distinct count.
-            #[expect(deprecated)]
-            let arguments = if is_distinct
-                && matches!(
-                    arguments.as_slice(),
-                    [expr::Expr::Wildcard {
-                        qualifier: None,
-                        options: _
-                    }]
-                ) {
-                schema
-                    .columns()
-                    .into_iter()
-                    .filter(|c| {
-                        state
-                            .get_field_info(&c.name)
-                            .is_ok_and(|info| !info.is_hidden())
-                    })
-                    .map(expr::Expr::Column)
-                    .collect()
-            } else {
-                arguments
-            };
-            let input = AggFunctionInput {
-                arguments,
-                distinct: is_distinct,
-                ignore_nulls,
-                filter,
-                order_by,
-                function_context: FunctionContextInput {
-                    argument_display_names: &argument_display_names,
-                    plan_config: &self.config,
-                    session_context: self.ctx,
-                    schema,
-                },
-            };
-            func(input)?
-        } else {
-            return Err(PlanError::unsupported(format!(
-                "unknown function: {function_name}",
-            )));
         };
 
         // DataFusion lambda variables carry no type until resolved against the schema.
@@ -445,14 +462,13 @@ impl PlanResolver<'_> {
         ];
         if arguments.len() >= 3
             && DATE_PART_FUNCTIONS.contains(&function_name.trim().to_lowercase().as_str())
+            && let spec::Expr::UnresolvedAttribute { ref name, .. } = arguments[0]
         {
-            if let spec::Expr::UnresolvedAttribute { ref name, .. } = arguments[0] {
-                let parts: Vec = name.clone().into();
-                if parts.len() == 1 {
-                    arguments[0] = spec::Expr::Literal(spec::Literal::Utf8 {
-                        value: Some(parts.into_iter().next().unwrap_or_default()),
-                    });
-                }
+            let parts: Vec = name.clone().into();
+            if parts.len() == 1 {
+                arguments[0] = spec::Expr::Literal(spec::Literal::Utf8 {
+                    value: Some(parts.into_iter().next().unwrap_or_default()),
+                });
             }
         }
         arguments
@@ -503,13 +519,12 @@ fn extract_metadata_from_udf(
     };
 
     // Try to extract metadata, but don't fail if it doesn't work
-    if let Ok(field) = udf.return_field_from_args(return_field_args) {
-        Ok(field
+    match udf.return_field_from_args(return_field_args) {
+        Ok(field) => Ok(field
             .metadata()
             .iter()
             .map(|(k, v)| (k.clone(), v.clone()))
-            .collect())
-    } else {
-        Ok(vec![])
+            .collect()),
+        _ => Ok(vec![]),
     }
 }
diff --git a/crates/sail-plan/src/resolver/expression/grouping.rs b/crates/sail-plan/src/resolver/expression/grouping.rs
index 7d261c3a2..8c6b18f4b 100644
--- a/crates/sail-plan/src/resolver/expression/grouping.rs
+++ b/crates/sail-plan/src/resolver/expression/grouping.rs
@@ -6,9 +6,9 @@ use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_rollup(
diff --git a/crates/sail-plan/src/resolver/expression/lambda.rs b/crates/sail-plan/src/resolver/expression/lambda.rs
index 20ae34b44..f8bd512f1 100644
--- a/crates/sail-plan/src/resolver/expression/lambda.rs
+++ b/crates/sail-plan/src/resolver/expression/lambda.rs
@@ -1,16 +1,16 @@
+use datafusion_common::DFSchemaRef;
 use datafusion_common::arrow::datatypes::FieldRef;
 use datafusion_common::datatype::FieldExt;
-use datafusion_common::DFSchemaRef;
 use datafusion_expr::expr::{Lambda, LambdaVariable};
-use datafusion_expr::{expr, ExprSchemable, ValueOrLambda};
+use datafusion_expr::{ExprSchemable, ValueOrLambda, expr};
 use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
 use crate::function::get_lambda_parameters;
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 pub(super) fn is_spec_lambda_argument(argument: &spec::Expr) -> bool {
     match argument {
diff --git a/crates/sail-plan/src/resolver/expression/literal.rs b/crates/sail-plan/src/resolver/expression/literal.rs
index b141a9728..309481ebb 100644
--- a/crates/sail-plan/src/resolver/expression/literal.rs
+++ b/crates/sail-plan/src/resolver/expression/literal.rs
@@ -12,9 +12,9 @@ use sail_sql_analyzer::parser::{parse_date, parse_time, parse_timestamp};
 
 use crate::config::DefaultTimestampType;
 use crate::error::PlanResult;
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) fn resolve_expression_literal(
@@ -76,6 +76,10 @@ impl PlanResolver<'_> {
                 (datetime.and_utc(), spec::TimestampType::WithoutTimeZone)
             }
         };
+        // Convert to microseconds.
+        // The parser accepts up to 9 digits (nanoseconds) for compatibility with Spark SQL syntax,
+        // but Spark stores timestamps with microsecond precision (6 digits).
+        // Nanoseconds beyond microsecond precision are truncated to match Spark behavior.
         let literal = spec::Literal::TimestampMicrosecond {
             microseconds: Some(datetime.timestamp_micros()),
             timestamp_type,
@@ -93,8 +97,10 @@ impl PlanResolver<'_> {
         // Convert to NaiveTime which validates hour (0-23), minute/second (0-59)
         let naive_time: NaiveTime = time.try_into()?;
 
-        // Use chrono methods to get microseconds since midnight
-        // Nanoseconds beyond microsecond precision are truncated to match Spark behavior
+        // Convert to microseconds since midnight.
+        // The parser accepts up to 9 digits (nanoseconds) for compatibility with Spark SQL syntax,
+        // but Spark stores timestamps with microsecond precision (6 digits).
+        // Nanoseconds beyond microsecond precision are truncated to match Spark behavior.
         let seconds_from_midnight = naive_time.num_seconds_from_midnight() as i64;
         let nanoseconds = naive_time.nanosecond() as i64;
         let microseconds = seconds_from_midnight * 1_000_000 + nanoseconds / 1_000;
diff --git a/crates/sail-plan/src/resolver/expression/misc.rs b/crates/sail-plan/src/resolver/expression/misc.rs
index a93dc75e8..fd946d2a8 100644
--- a/crates/sail-plan/src/resolver/expression/misc.rs
+++ b/crates/sail-plan/src/resolver/expression/misc.rs
@@ -3,25 +3,26 @@ use std::sync::Arc;
 
 use arrow::datatypes::DataType;
 use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion_common::{plan_datafusion_err, Column, DFSchemaRef, ScalarValue};
+use datafusion_common::{Column, DFSchemaRef, ScalarValue, plan_datafusion_err};
 use datafusion_expr::expr::FieldMetadata;
-use datafusion_expr::{expr, lit, BinaryExpr, ExprSchemable, ScalarUDF};
-use datafusion_expr_common::operator::Operator;
+use datafusion_expr::{ExprSchemable, ScalarUDF, cast, expr, lit, when};
 use datafusion_functions::core::expr_ext::FieldAccessor;
-use datafusion_functions_nested::expr_fn::{array_element, map_extract};
+use datafusion_functions::expr_fn as datafusion_fn;
+use datafusion_functions_nested::expr_fn::{array_element, array_length, map_extract};
 use sail_common::spec::{self, DEFAULT_COLUMN_VALUE_PLACEHOLDER_ID};
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::literal::LiteralEvaluator;
 use sail_common_datafusion::session::plan::PlanService;
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::drop_struct_field::DropStructField;
+use sail_function::scalar::misc::raise_error::RaiseError;
 use sail_function::scalar::table_input::TableInput;
 use sail_function::scalar::update_struct_field::UpdateStructField;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_alias(
@@ -117,15 +118,11 @@ impl PlanResolver<'_> {
                         return Ok(Transformed::yes(expr::Expr::Literal(scalar.clone(), None)));
                     }
                     // Try positional parameter (key is a 1-based integer index).
-                    if let Ok(index) = key.parse::() {
-                        if index > 0 {
-                            if let Some(scalar) = state.get_positional_param_value(index - 1) {
-                                return Ok(Transformed::yes(expr::Expr::Literal(
-                                    scalar.clone(),
-                                    None,
-                                )));
-                            }
-                        }
+                    if let Ok(index) = key.parse::()
+                        && index > 0
+                        && let Some(scalar) = state.get_positional_param_value(index - 1)
+                    {
+                        return Ok(Transformed::yes(expr::Expr::Literal(scalar.clone(), None)));
                     }
                 }
                 Ok(Transformed::no(e))
@@ -303,18 +300,46 @@ impl PlanResolver<'_> {
             }
         };
         let expr = match data_type {
-            DataType::List(_)
-            | DataType::LargeList(_)
-            | DataType::FixedSizeList(_, _)
-            | DataType::ListView(_)
-            | DataType::LargeListView(_) => array_element(
-                expr,
-                expr::Expr::BinaryExpr(BinaryExpr::new(
-                    Box::new(expr::Expr::Literal(extraction, None)),
-                    Operator::Plus,
-                    Box::new(lit(1i64)),
-                )),
-            ),
+            DataType::List(field)
+            | DataType::LargeList(field)
+            | DataType::FixedSizeList(field, _)
+            | DataType::ListView(field)
+            | DataType::LargeListView(field) => {
+                let ScalarValue::Int64(index) = extraction.cast_to(&DataType::Int64)? else {
+                    return Err(PlanError::AnalysisError(format!(
+                        "invalid extraction value for array: {extraction}"
+                    )));
+                };
+                let index_expr = lit(ScalarValue::Int64(index));
+                let element = array_element(
+                    expr.clone(),
+                    lit(ScalarValue::Int64(index.map(|x| x.saturating_add(1)))),
+                );
+                if self.config.ansi_mode {
+                    let length = cast(array_length(expr.clone()), DataType::Int64);
+                    let message = datafusion_fn::concat(vec![
+                        lit("[INVALID_ARRAY_INDEX] The index "),
+                        cast(index_expr, DataType::Utf8),
+                        lit(" is out of bounds. The array has "),
+                        cast(length.clone(), DataType::Utf8),
+                        lit(
+                            " elements. Use the SQL function `get()` to tolerate accessing element at invalid index and return NULL instead.",
+                        ),
+                    ]);
+                    let error = cast(
+                        ScalarUDF::from(RaiseError::new()).call(vec![message]),
+                        field.data_type().clone(),
+                    );
+                    let out_of_bounds = match index {
+                        Some(index) if index < 0 => expr.is_not_null(),
+                        Some(index) => expr.is_not_null().and(length.lt_eq(lit(index))),
+                        None => lit(false),
+                    };
+                    when(out_of_bounds, error).otherwise(element)?
+                } else {
+                    element
+                }
+            }
             DataType::Struct(fields) => {
                 let ScalarValue::Utf8(Some(name)) = extraction else {
                     return Err(PlanError::AnalysisError(format!(
@@ -337,7 +362,7 @@ impl PlanResolver<'_> {
             _ => {
                 return Err(PlanError::AnalysisError(format!(
                     "cannot extract value from data type: {data_type}"
-                )))
+                )));
             }
         };
         Ok(NamedExpr::new(vec![name], expr))
@@ -364,15 +389,35 @@ impl PlanResolver<'_> {
             )));
         };
 
-        let new_expr = if let Some(value_expression) = value_expression {
-            let value_expr = self
-                .resolve_expression(value_expression, schema, state)
+        // Spark names the column after the `UpdateFields` expression tree, where
+        // each operation is rendered as `WithField()` (the value
+        // expression's display name, not the target field name) or `dropfield()`.
+        let (op, new_expr) = if let Some(value_expression) = value_expression {
+            let NamedExpr {
+                name: value_name,
+                expr: value_expr,
+                ..
+            } = self
+                .resolve_named_expression(value_expression, schema, state)
                 .await?;
-            ScalarUDF::from(UpdateStructField::new(field_name)).call(vec![expr, value_expr])
+            (
+                format!("WithField({})", value_name.one()?),
+                ScalarUDF::from(UpdateStructField::new(field_name)).call(vec![expr, value_expr]),
+            )
         } else {
-            ScalarUDF::from(DropStructField::new(field_name)).call(vec![expr])
+            (
+                "dropfield()".to_string(),
+                ScalarUDF::from(DropStructField::new(field_name)).call(vec![expr]),
+            )
+        };
+        // Spark collapses chained `withField`/`dropFields` into a single
+        // `update_fields(x, op1, op2, ...)`, so splice the new operation into an
+        // existing `update_fields(...)` name rather than nesting.
+        let result_name = match name.strip_suffix(')') {
+            Some(prefix) if prefix.starts_with("update_fields(") => format!("{prefix}, {op})"),
+            _ => format!("update_fields({name}, {op})"),
         };
-        Ok(NamedExpr::new(vec![name], new_expr))
+        Ok(NamedExpr::new(vec![result_name], new_expr))
     }
 
     /// Rewrites the resolved expression to refer to columns in an external schema.
diff --git a/crates/sail-plan/src/resolver/expression/mod.rs b/crates/sail-plan/src/resolver/expression/mod.rs
index 0955ca90c..6e1d2ceb5 100644
--- a/crates/sail-plan/src/resolver/expression/mod.rs
+++ b/crates/sail-plan/src/resolver/expression/mod.rs
@@ -7,8 +7,8 @@ use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 mod attribute;
 mod cast;
@@ -215,20 +215,19 @@ impl PlanResolver<'_> {
                 // Detect multi-column IN subquery: (a, b) IN (SELECT x, y FROM ...)
                 // The SQL parser produces a Tuple which the analyzer converts to
                 // UnresolvedFunction("struct", [a, b]).
-                if let Expr::UnresolvedFunction(ref f) = *expr {
-                    if f.function_name.parts() == [spec::Identifier::from("struct")]
-                        && f.arguments.len() > 1
-                    {
-                        let arguments = match *expr {
-                            Expr::UnresolvedFunction(f) => f.arguments,
-                            _ => unreachable!(),
-                        };
-                        return self
-                            .resolve_multi_column_in_subquery(
-                                arguments, *subquery, negated, schema, state,
-                            )
-                            .await;
-                    }
+                if let Expr::UnresolvedFunction(ref f) = *expr
+                    && f.function_name.parts() == [spec::Identifier::from("struct")]
+                    && f.arguments.len() > 1
+                {
+                    let arguments = match *expr {
+                        Expr::UnresolvedFunction(f) => f.arguments,
+                        _ => unreachable!(),
+                    };
+                    return self
+                        .resolve_multi_column_in_subquery(
+                            arguments, *subquery, negated, schema, state,
+                        )
+                        .await;
                 }
                 self.resolve_expression_in_subquery(*expr, *subquery, negated, schema, state)
                     .await
@@ -424,9 +423,9 @@ mod tests {
     use crate::config::PlanConfig;
     use crate::error::PlanResult;
     use crate::formatter::SparkPlanFormatter;
+    use crate::resolver::PlanResolver;
     use crate::resolver::expression::NamedExpr;
     use crate::resolver::state::PlanResolverState;
-    use crate::resolver::PlanResolver;
 
     fn create_session() -> PlanResult {
         let mut state = SessionStateBuilder::new().build();
diff --git a/crates/sail-plan/src/resolver/expression/predicate.rs b/crates/sail-plan/src/resolver/expression/predicate.rs
index 48d8e293f..ff7885582 100644
--- a/crates/sail-plan/src/resolver/expression/predicate.rs
+++ b/crates/sail-plan/src/resolver/expression/predicate.rs
@@ -1,12 +1,12 @@
 use datafusion_common::DFSchemaRef;
-use datafusion_expr::{expr, BinaryExpr};
+use datafusion_expr::{BinaryExpr, expr};
 use datafusion_expr_common::operator::Operator;
 use sail_common::spec;
 
 use crate::error::PlanResult;
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     // TODO: Construct better names for the expression (e.g. a IN (b, c)) for all functions below.
diff --git a/crates/sail-plan/src/resolver/expression/sort.rs b/crates/sail-plan/src/resolver/expression/sort.rs
index dcb407e32..7f63c1c88 100644
--- a/crates/sail-plan/src/resolver/expression/sort.rs
+++ b/crates/sail-plan/src/resolver/expression/sort.rs
@@ -3,9 +3,9 @@ use datafusion_expr::expr;
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_sort_order(
@@ -54,7 +54,7 @@ impl PlanResolver<'_> {
                             expr: self.resolve_expression(*child, schema, state).await?,
                             asc,
                             nulls_first,
-                        })
+                        });
                     }
                 };
                 if position > 0 && position <= num_fields {
diff --git a/crates/sail-plan/src/resolver/expression/subquery.rs b/crates/sail-plan/src/resolver/expression/subquery.rs
index 9583af69f..65c7b3719 100644
--- a/crates/sail-plan/src/resolver/expression/subquery.rs
+++ b/crates/sail-plan/src/resolver/expression/subquery.rs
@@ -1,15 +1,15 @@
 use std::sync::Arc;
 
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
 use datafusion_common::DFSchemaRef;
+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
 use datafusion_expr::logical_plan::{Filter, Projection};
-use datafusion_expr::{expr_fn, Expr, LogicalPlan};
+use datafusion_expr::{Expr, LogicalPlan, expr_fn};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_in_subquery(
diff --git a/crates/sail-plan/src/resolver/expression/udf.rs b/crates/sail-plan/src/resolver/expression/udf.rs
index 0ccd8dd72..e77dd7017 100644
--- a/crates/sail-plan/src/resolver/expression/udf.rs
+++ b/crates/sail-plan/src/resolver/expression/udf.rs
@@ -4,7 +4,7 @@ use arrow::datatypes::DataType;
 use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
 use datafusion_common::{DFSchemaRef, DataFusionError};
 use datafusion_expr::expr::AggregateFunctionParams;
-use datafusion_expr::{expr, AggregateUDF, Expr, ExprSchemable, ScalarUDF};
+use datafusion_expr::{AggregateUDF, Expr, ExprSchemable, ScalarUDF, expr};
 use sail_common::spec;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::plan::PlanService;
@@ -14,10 +14,10 @@ use sail_python_udf::udf::pyspark_udaf::{PySparkGroupAggKind, PySparkGroupAggreg
 use sail_python_udf::udf::pyspark_udf::{PySparkUDF, PySparkUdfKind};
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::function::PythonUdf;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 /// If `expr` contains (or is) an `AggregateFunction`, return its name.
 /// Used to detect illegal nesting of aggregate functions as UDAF arguments.
diff --git a/crates/sail-plan/src/resolver/expression/wildcard.rs b/crates/sail-plan/src/resolver/expression/wildcard.rs
index 9d0d8f5fc..eb3c5028f 100644
--- a/crates/sail-plan/src/resolver/expression/wildcard.rs
+++ b/crates/sail-plan/src/resolver/expression/wildcard.rs
@@ -4,17 +4,17 @@ use arrow::datatypes::DataType;
 use datafusion_common::{DFSchemaRef, TableReference};
 use datafusion_expr::expr::ScalarFunction;
 use datafusion_expr::sql::{Ident, ObjectName, ObjectNamePart};
-use datafusion_expr::{col, expr, lit, ScalarUDF};
+use datafusion_expr::{ScalarUDF, col, expr, lit};
 use datafusion_functions::core::get_field;
 use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::multi_expr::MultiExpr;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::expression::attribute::qualifier_matches;
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
+use crate::resolver::expression::attribute::qualifier_matches;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_wildcard(
diff --git a/crates/sail-plan/src/resolver/expression/window.rs b/crates/sail-plan/src/resolver/expression/window.rs
index c860a2243..693450bd9 100644
--- a/crates/sail-plan/src/resolver/expression/window.rs
+++ b/crates/sail-plan/src/resolver/expression/window.rs
@@ -7,7 +7,7 @@ use datafusion_common::{DFSchemaRef, DataFusionError, ScalarValue};
 use datafusion_expr::expr::WindowFunctionParams;
 use datafusion_expr::simplify::SimplifyContextBuilder;
 use datafusion_expr::{
-    expr, AggregateUDF, ExprSchemable, WindowFrame, WindowFrameBound, WindowFrameUnits,
+    AggregateUDF, ExprSchemable, WindowFrame, WindowFrameBound, WindowFrameUnits, expr,
 };
 use sail_catalog::manager::CatalogManager;
 use sail_common::spec::{self};
@@ -21,11 +21,11 @@ use sail_python_udf::udf::pyspark_udaf::{PySparkGroupAggKind, PySparkGroupAggreg
 use sail_python_udf::udf::pyspark_unresolved_udf::PySparkUnresolvedUDF;
 
 use crate::error::{PlanError, PlanResult};
-use crate::function::common::{get_null_treatment, FunctionContextInput, WinFunctionInput};
+use crate::function::common::{FunctionContextInput, WinFunctionInput, get_null_treatment};
 use crate::function::get_built_in_window_function;
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_expression_window(
@@ -243,7 +243,7 @@ impl PlanResolver<'_> {
                     _ => {
                         return Err(PlanError::invalid(
                             "invalid user-defined window function type",
-                        ))
+                        ));
                     }
                 };
                 let window = expr::Expr::WindowFunction(Box::new(expr::WindowFunction {
diff --git a/crates/sail-plan/src/resolver/function.rs b/crates/sail-plan/src/resolver/function.rs
index e8c237bfd..840e1fd9c 100644
--- a/crates/sail-plan/src/resolver/function.rs
+++ b/crates/sail-plan/src/resolver/function.rs
@@ -3,8 +3,8 @@ use datafusion_common::plan_err;
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 pub(super) struct PythonUdf {
     pub python_version: String,
diff --git a/crates/sail-plan/src/resolver/literal.rs b/crates/sail-plan/src/resolver/literal.rs
index 83ce84ef9..721874a11 100644
--- a/crates/sail-plan/src/resolver/literal.rs
+++ b/crates/sail-plan/src/resolver/literal.rs
@@ -1,21 +1,21 @@
 use std::sync::Arc;
 
 use datafusion::arrow::array::{
-    new_empty_array, new_null_array, ArrayData, AsArray, FixedSizeListArray, LargeListArray,
-    MapArray, StructArray,
+    ArrayData, AsArray, FixedSizeListArray, LargeListArray, MapArray, StructArray, new_empty_array,
+    new_null_array,
 };
 use datafusion::arrow::buffer::OffsetBuffer;
 use datafusion::arrow::datatypes as adt;
+use datafusion_common::ScalarValue;
 use datafusion_common::scalar::ScalarStructBuilder;
 use datafusion_common::utils::SingleRowListArrayBuilder;
-use datafusion_common::ScalarValue;
 use sail_common::spec::{
     self, Literal, SAIL_MAP_FIELD_NAME, SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME,
 };
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) fn resolve_literal(
diff --git a/crates/sail-plan/src/resolver/plan.rs b/crates/sail-plan/src/resolver/plan.rs
index 79f32cb42..9310c57e2 100644
--- a/crates/sail-plan/src/resolver/plan.rs
+++ b/crates/sail-plan/src/resolver/plan.rs
@@ -2,8 +2,8 @@ use datafusion::logical_expr::LogicalPlan;
 use sail_common::spec;
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 #[derive(Debug)]
 pub struct NamedPlan {
@@ -20,11 +20,13 @@ impl PlanResolver<'_> {
         match plan {
             spec::Plan::Query(query) => {
                 let plan = self.resolve_query_plan(query, &mut state).await?;
+                let plan = Self::preserve_order_sensitive_aggregate_sorts(plan)?;
                 let fields = Some(Self::get_field_names(plan.schema(), &state)?);
                 Ok(NamedPlan { plan, fields })
             }
             spec::Plan::Command(command) => {
                 let plan = self.resolve_command_plan(command, &mut state).await?;
+                let plan = Self::preserve_order_sensitive_aggregate_sorts(plan)?;
                 Ok(NamedPlan { plan, fields: None })
             }
         }
diff --git a/crates/sail-plan/src/resolver/query/aggregate.rs b/crates/sail-plan/src/resolver/query/aggregate.rs
index c321701bd..7cc3f5e99 100644
--- a/crates/sail-plan/src/resolver/query/aggregate.rs
+++ b/crates/sail-plan/src/resolver/query/aggregate.rs
@@ -1,26 +1,37 @@
+use std::sync::Arc;
+
+use datafusion::functions_aggregate::{average, bit_and_or_xor, bool_and_or, count, min_max, sum};
+use datafusion_common::arrow::datatypes::DataType;
 use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion};
 use datafusion_common::{
     Column, DFSchemaRef, DataFusionError, Result as DataFusionResult, ScalarValue,
 };
-use datafusion_expr::utils::{expr_as_column_expr, find_aggregate_exprs};
+use datafusion_expr::expr_rewriter::normalize_col;
+use datafusion_expr::logical_plan::{FetchType, SkipType};
+use datafusion_expr::utils::find_aggregate_exprs;
 use datafusion_expr::{
-    bitwise_and, bitwise_shift_right, cast, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder,
-    Volatility,
+    Aggregate, AggregateUDF, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder,
+    Projection, SortExpr, Volatility, bitwise_and, bitwise_shift_right, cast,
 };
+use datafusion_spark::function::aggregate::try_sum::SparkTrySum;
 use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
+use sail_function::aggregate::try_avg::TryAvgFunction;
 use sail_function::scalar::explode::Explode;
+use sail_logical_plan::monotonic_id::MonotonicIdNode;
+use sail_logical_plan::sort::{RequiredSortNode, SortWithinPartitionsNode};
+use sail_logical_plan::spark_partition_id::SparkPartitionIdNode;
 use sail_python_udf::get_udf_display_name;
 use sail_python_udf::udf::pyspark_udaf::PySparkGroupAggregateUDF;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::{AggregateState, PlanResolverState};
 use crate::resolver::tree::explode::ExplodeRewriter;
 use crate::resolver::tree::monotonic_id::MonotonicIdRewriter;
 use crate::resolver::tree::spark_partition_id::SparkPartitionIdRewriter;
 use crate::resolver::tree::window::WindowRewriter;
-use crate::resolver::PlanResolver;
 
 /// Projections resolved by index (a `None` marks a projection deferred until the
 /// grouping is materialized), paired with the indices of the deferred projections.
@@ -36,11 +47,11 @@ type GeneratorReplacements = Vec<(Expr, Expr)>;
 fn find_volatile_in_aggregate_context(expr: &Expr) -> Option {
     let mut found_name: Option = None;
     let _ = expr.apply(|e| {
-        if let Expr::ScalarFunction(f) = e {
-            if f.func.signature().volatility == Volatility::Volatile {
-                found_name = Some(f.func.name().to_string());
-                return Ok(TreeNodeRecursion::Stop);
-            }
+        if let Expr::ScalarFunction(f) = e
+            && f.func.signature().volatility == Volatility::Volatile
+        {
+            found_name = Some(f.func.name().to_string());
+            return Ok(TreeNodeRecursion::Stop);
         }
         Ok(TreeNodeRecursion::Continue)
     });
@@ -279,6 +290,381 @@ impl PlanResolver<'_> {
             .build()?)
     }
 
+    pub(crate) fn input_sort_ordering(input: &LogicalPlan) -> Option> {
+        Self::find_input_sort_ordering(input)?
+            .into_iter()
+            .map(|sort| {
+                Some(SortExpr {
+                    expr: normalize_col(sort.expr, input).ok()?,
+                    asc: sort.asc,
+                    nulls_first: sort.nulls_first,
+                })
+            })
+            .collect()
+    }
+
+    fn find_input_sort_ordering(input: &LogicalPlan) -> Option> {
+        match input {
+            LogicalPlan::Sort(sort) => Some(sort.expr.clone()),
+            LogicalPlan::SubqueryAlias(alias) => {
+                let ordering = Self::find_input_sort_ordering(alias.input.as_ref())?;
+                ordering
+                    .into_iter()
+                    .map(|sort| {
+                        let expr = sort
+                            .expr
+                            .transform(|expr| match expr {
+                                Expr::Column(mut column) => {
+                                    column.relation = Some(alias.alias.clone());
+                                    Ok(Transformed::yes(Expr::Column(column)))
+                                }
+                                expr => Ok(Transformed::no(expr)),
+                            })
+                            .data()
+                            .ok()?;
+                        Some(SortExpr { expr, ..sort })
+                    })
+                    .collect()
+            }
+            LogicalPlan::Filter(filter) => Self::find_input_sort_ordering(filter.input.as_ref()),
+            LogicalPlan::Limit(limit) => Self::find_input_sort_ordering(limit.input.as_ref()),
+            LogicalPlan::Projection(projection) => {
+                let ordering = Self::find_input_sort_ordering(projection.input.as_ref())?;
+                ordering
+                    .into_iter()
+                    .map(|sort| {
+                        let expr = Self::remap_through_projection(sort.expr, projection)?;
+                        Some(SortExpr { expr, ..sort })
+                    })
+                    .collect()
+            }
+            LogicalPlan::Window(window) => {
+                let [Expr::WindowFunction(function)] = window.window_expr.as_slice() else {
+                    return None;
+                };
+                let mut required = function
+                    .params
+                    .partition_by
+                    .iter()
+                    .cloned()
+                    .map(|expr| SortExpr {
+                        expr,
+                        asc: true,
+                        nulls_first: true,
+                    })
+                    .collect::>();
+                required.extend(function.params.order_by.clone());
+
+                let child = Self::find_input_sort_ordering(window.input.as_ref());
+                if required.is_empty() {
+                    child
+                } else {
+                    match child {
+                        Some(child) if child.starts_with(&required) => Some(child),
+                        _ => Some(required),
+                    }
+                }
+            }
+            LogicalPlan::Unnest(unnest) => {
+                let ordering = Self::find_input_sort_ordering(unnest.input.as_ref())?;
+                let references_unnested_column = ordering.iter().any(|sort| {
+                    sort.expr.column_refs().iter().any(|column| {
+                        unnest
+                            .input
+                            .schema()
+                            .maybe_index_of_column(column)
+                            .is_none_or(|index| {
+                                unnest
+                                    .list_type_columns
+                                    .iter()
+                                    .any(|(unnested, _)| *unnested == index)
+                                    || unnest.struct_type_columns.contains(&index)
+                            })
+                    })
+                });
+                (!references_unnested_column).then_some(ordering)
+            }
+            LogicalPlan::Extension(extension) => {
+                let node = extension.node.as_any();
+                if let Some(sort) = node.downcast_ref::() {
+                    Some(sort.sort_expr().to_vec())
+                } else if let Some(sort) = node.downcast_ref::() {
+                    Some(sort.sort_expr().to_vec())
+                } else if let Some(node) = node.downcast_ref::() {
+                    Self::find_input_sort_ordering(node.input().as_ref())
+                } else if let Some(node) = node.downcast_ref::() {
+                    Self::find_input_sort_ordering(node.input().as_ref())
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
+    fn remap_through_projection(expr: Expr, projection: &Projection) -> Option {
+        let rewritten = expr
+            .transform_down(|expr| {
+                match projection
+                    .expr
+                    .iter()
+                    .position(|projection| projection.clone().unalias() == expr)
+                {
+                    Some(index) => {
+                        let (qualifier, field) = projection.schema.qualified_field(index);
+                        Ok(Transformed::new(
+                            Expr::from(Column::from((qualifier, field))),
+                            true,
+                            TreeNodeRecursion::Jump,
+                        ))
+                    }
+                    None => Ok(Transformed::no(expr)),
+                }
+            })
+            .data()
+            .ok()?;
+        let mut derivable = true;
+        let _ = rewritten.apply(|expr| {
+            if let Expr::Column(column) = expr
+                && projection
+                    .schema
+                    .qualified_field_from_column(column)
+                    .is_err()
+            {
+                derivable = false;
+                return Ok(TreeNodeRecursion::Stop);
+            }
+            Ok(TreeNodeRecursion::Continue)
+        });
+        derivable.then_some(rewritten)
+    }
+
+    pub(crate) fn is_order_irrelevant_udaf(
+        udf: &AggregateUDF,
+        args: &[Expr],
+        schema: &DFSchemaRef,
+    ) -> PlanResult {
+        if udf == min_max::min_udaf().as_ref()
+            || udf == min_max::max_udaf().as_ref()
+            || udf == count::count_udaf().as_ref()
+            || udf == bit_and_or_xor::bit_and_udaf().as_ref()
+            || udf == bit_and_or_xor::bit_or_udaf().as_ref()
+            || udf == bit_and_or_xor::bit_xor_udaf().as_ref()
+            || udf == bool_and_or::bool_and_udaf().as_ref()
+            || udf == bool_and_or::bool_or_udaf().as_ref()
+        {
+            return Ok(true);
+        }
+
+        if udf == sum::sum_udaf().as_ref()
+            || udf == average::avg_udaf().as_ref()
+            || udf.inner().is::()
+            || udf.inner().is::()
+        {
+            let Some(arg) = args.first() else {
+                return Ok(false);
+            };
+            return Ok(!matches!(
+                arg.get_type(schema.as_ref())?,
+                DataType::Float16 | DataType::Float32 | DataType::Float64
+            ));
+        }
+
+        Ok(false)
+    }
+
+    fn requires_input_order(
+        aggregates: &[Expr],
+        schema: &DFSchemaRef,
+        has_grouping: bool,
+    ) -> PlanResult {
+        for aggregate in aggregates {
+            let Expr::AggregateFunction(function) = aggregate else {
+                return Ok(true);
+            };
+            let udf = function.func.as_ref();
+            if !has_grouping
+                && (udf == sum::sum_udaf().as_ref() || udf == average::avg_udaf().as_ref())
+            {
+                continue;
+            }
+            if !Self::is_order_irrelevant_udaf(udf, &function.params.args, schema)? {
+                return Ok(true);
+            }
+        }
+        Ok(false)
+    }
+
+    pub(crate) fn preserve_order_sensitive_aggregate_sorts(
+        plan: LogicalPlan,
+    ) -> PlanResult {
+        Ok(plan
+            .transform_up_with_subqueries(|plan| {
+                let LogicalPlan::Aggregate(mut aggregate) = plan else {
+                    return Ok(Transformed::no(plan));
+                };
+                let aggregate_exprs = find_aggregate_exprs(&aggregate.aggr_expr);
+                if !Self::requires_input_order(
+                    &aggregate_exprs,
+                    aggregate.input.schema(),
+                    !aggregate.group_expr.is_empty(),
+                )
+                .map_err(|error| DataFusionError::External(Box::new(error)))?
+                {
+                    return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
+                }
+
+                let (input, found, _) = Self::require_input_sort_inner(Arc::unwrap_or_clone(
+                    Arc::clone(&aggregate.input),
+                ))
+                .map_err(|error| DataFusionError::External(Box::new(error)))?;
+                if !found {
+                    return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
+                }
+                aggregate.input = Arc::new(input);
+                Ok(Transformed::yes(LogicalPlan::Aggregate(aggregate)))
+            })
+            .data()?)
+    }
+
+    pub(crate) fn require_input_sort_inner(
+        plan: LogicalPlan,
+    ) -> PlanResult<(LogicalPlan, bool, bool)> {
+        match plan {
+            LogicalPlan::Sort(sort) => Ok((
+                LogicalPlan::Extension(Extension {
+                    node: Arc::new(RequiredSortNode::new(
+                        sort.input, sort.expr, sort.fetch, false,
+                    )),
+                }),
+                true,
+                true,
+            )),
+            LogicalPlan::Limit(mut limit) => {
+                let fetch = match (limit.get_skip_type()?, limit.get_fetch_type()?) {
+                    (SkipType::Literal(skip), FetchType::Literal(Some(fetch))) => Some(
+                        skip.checked_add(fetch)
+                            .ok_or_else(|| PlanError::invalid("LIMIT + OFFSET overflow"))?,
+                    ),
+                    _ => None,
+                };
+                let input = Arc::unwrap_or_clone(std::mem::replace(
+                    &mut limit.input,
+                    Arc::new(LogicalPlan::default()),
+                ));
+                let (input, found, global) = match fetch {
+                    Some(fetch) => Self::require_input_sort_with_fetch(input, fetch)?,
+                    None => Self::require_input_sort_inner(input)?,
+                };
+                limit.input = Arc::new(input);
+                Ok((LogicalPlan::Limit(limit), found, global))
+            }
+            LogicalPlan::Extension(extension) => {
+                if let Some(sort) = extension.node.as_any().downcast_ref::() {
+                    let global = !sort.preserve_partitioning();
+                    return Ok((LogicalPlan::Extension(extension), true, global));
+                }
+                if let Some(sort) = extension
+                    .node
+                    .as_any()
+                    .downcast_ref::()
+                {
+                    return Ok((
+                        LogicalPlan::Extension(Extension {
+                            node: Arc::new(RequiredSortNode::new(
+                                Arc::clone(sort.input()),
+                                sort.sort_expr().to_vec(),
+                                sort.fetch(),
+                                true,
+                            )),
+                        }),
+                        true,
+                        false,
+                    ));
+                }
+                if !extension.node.as_any().is::()
+                    && !extension.node.as_any().is::()
+                {
+                    return Ok((LogicalPlan::Extension(extension), false, false));
+                }
+
+                let plan = LogicalPlan::Extension(extension);
+                let expressions = plan.expressions();
+                let child = plan.inputs().one()?.clone();
+                let (child, found, global) = Self::require_input_sort_inner(child)?;
+                if found {
+                    Ok((plan.with_new_exprs(expressions, vec![child])?, true, global))
+                } else {
+                    Ok((plan, false, false))
+                }
+            }
+            plan => {
+                let transparent = matches!(
+                    plan,
+                    LogicalPlan::SubqueryAlias(_)
+                        | LogicalPlan::Projection(_)
+                        | LogicalPlan::Filter(_)
+                        | LogicalPlan::Window(_)
+                        | LogicalPlan::Unnest(_)
+                );
+                if !transparent {
+                    return Ok((plan, false, false));
+                }
+
+                // Unnest's with_new_exprs requires no expressions and rebuilds from its own exec_columns.
+                let expressions = if matches!(&plan, LogicalPlan::Unnest(_)) {
+                    vec![]
+                } else {
+                    plan.expressions()
+                };
+                let child = plan.inputs().one()?.clone();
+                let (child, found, global) = Self::require_input_sort_inner(child)?;
+                if found {
+                    Ok((plan.with_new_exprs(expressions, vec![child])?, true, global))
+                } else {
+                    Ok((plan, false, false))
+                }
+            }
+        }
+    }
+
+    fn require_input_sort_with_fetch(
+        plan: LogicalPlan,
+        fetch: usize,
+    ) -> PlanResult<(LogicalPlan, bool, bool)> {
+        match plan {
+            LogicalPlan::Sort(sort) => Ok((
+                LogicalPlan::Extension(Extension {
+                    node: Arc::new(RequiredSortNode::new(
+                        sort.input,
+                        sort.expr,
+                        Some(sort.fetch.map_or(fetch, |old| old.min(fetch))),
+                        false,
+                    )),
+                }),
+                true,
+                true,
+            )),
+            plan => {
+                if !matches!(
+                    plan,
+                    LogicalPlan::Projection(_) | LogicalPlan::SubqueryAlias(_)
+                ) {
+                    return Self::require_input_sort_inner(plan);
+                }
+
+                let expressions = plan.expressions();
+                let child = plan.inputs().one()?.clone();
+                let (child, found, global) = Self::require_input_sort_with_fetch(child, fetch)?;
+                if found {
+                    Ok((plan.with_new_exprs(expressions, vec![child])?, true, global))
+                } else {
+                    Ok((plan, false, false))
+                }
+            }
+        }
+    }
+
     pub(super) fn has_grouping_set(grouping: &[Expr]) -> bool {
         grouping.iter().any(|x| matches!(x, Expr::GroupingSet(_)))
     }
@@ -598,7 +984,10 @@ impl PlanResolver<'_> {
         Ok(expr
             .transform_down(|e| {
                 if base.contains(&e) {
-                    Ok(Transformed::yes(expr_as_column_expr(&e, plan)?))
+                    Ok(Transformed::yes(
+                        Self::expr_as_column_expr(&e, plan)
+                            .map_err(|e| DataFusionError::External(Box::new(e)))?,
+                    ))
                 } else {
                     Ok(Transformed::no(e))
                 }
@@ -606,6 +995,26 @@ impl PlanResolver<'_> {
             .data()?)
     }
 
+    // Modification of DataFusion's `expr_as_column_expr`
+    fn expr_as_column_expr(expr: &Expr, plan: &LogicalPlan) -> PlanResult {
+        match expr {
+            Expr::Column(column) => {
+                let result = plan
+                    .schema()
+                    .qualified_field_from_column(column)
+                    .or_else(|_| {
+                        let column = Column::new_unqualified(column.name.clone());
+                        plan.schema().qualified_field_from_column(&column)
+                    })?;
+                let (qualifier, field) = result;
+                Ok(Expr::from(Column::from((qualifier, field))))
+            }
+            _ => Ok(Expr::Column(Column::from_name(
+                expr.schema_name().to_string(),
+            ))),
+        }
+    }
+
     /// Spark CheckAnalysis: GroupedAgg Pandas/Arrow UDFs cannot be mixed with regular
     /// (non-UDF) aggregate functions in the same `.agg()` call.
     fn check_no_mixed_grouped_agg_udf(projections: &[NamedExpr]) -> PlanResult<()> {
@@ -628,14 +1037,14 @@ impl PlanResolver<'_> {
                 Ok(TreeNodeRecursion::Continue)
             });
         }
-        if let Some(udf_name) = pyspark_agg_name {
-            if has_regular_agg {
-                return Err(PlanError::AnalysisError(format!(
-                    // Spark tests expect this error message. Typo is intended.
-                    "The group aggregate pandas UDF `{udf_name}` cannot be invoked \
+        if let Some(udf_name) = pyspark_agg_name
+            && has_regular_agg
+        {
+            return Err(PlanError::AnalysisError(format!(
+                // Spark tests expect this error message. Typo is intended.
+                "The group aggregate pandas UDF `{udf_name}` cannot be invoked \
                      together with as other, non-pandas aggregate functions."
-                )));
-            }
+            )));
         }
         Ok(())
     }
diff --git a/crates/sail-plan/src/resolver/query/alias.rs b/crates/sail-plan/src/resolver/query/alias.rs
index c9294e153..a2b85139a 100644
--- a/crates/sail-plan/src/resolver/query/alias.rs
+++ b/crates/sail-plan/src/resolver/query/alias.rs
@@ -5,8 +5,8 @@ use datafusion_expr::{Expr, LogicalPlan, Projection, SubqueryAlias};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_subquery_alias(
diff --git a/crates/sail-plan/src/resolver/query/column_op.rs b/crates/sail-plan/src/resolver/query/column_op.rs
index f8ed9e100..284f93380 100644
--- a/crates/sail-plan/src/resolver/query/column_op.rs
+++ b/crates/sail-plan/src/resolver/query/column_op.rs
@@ -3,20 +3,20 @@ use std::sync::Arc;
 
 use datafusion_common::Column;
 use datafusion_expr::{
-    cast, col, lit, Expr, ExprSchemable, LogicalPlan, Projection, SubqueryAlias,
+    Expr, ExprSchemable, LogicalPlan, Projection, SubqueryAlias, cast, col, lit,
 };
 use indexmap::IndexMap;
 use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
 use crate::resolver::tree::explode::ExplodeRewriter;
 use crate::resolver::tree::monotonic_id::MonotonicIdRewriter;
 use crate::resolver::tree::spark_partition_id::SparkPartitionIdRewriter;
 use crate::resolver::tree::window::WindowRewriter;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_to_df(
diff --git a/crates/sail-plan/src/resolver/query/cte.rs b/crates/sail-plan/src/resolver/query/cte.rs
index 718f92b81..19f102e34 100644
--- a/crates/sail-plan/src/resolver/query/cte.rs
+++ b/crates/sail-plan/src/resolver/query/cte.rs
@@ -5,8 +5,8 @@ use datafusion_expr::{LogicalPlan, SubqueryAlias};
 use sail_common::spec;
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_with_ctes(
diff --git a/crates/sail-plan/src/resolver/query/dedup.rs b/crates/sail-plan/src/resolver/query/dedup.rs
index c8f9efcc3..d3f40f956 100644
--- a/crates/sail-plan/src/resolver/query/dedup.rs
+++ b/crates/sail-plan/src/resolver/query/dedup.rs
@@ -4,8 +4,8 @@ use datafusion_expr::{Distinct, DistinctOn, Expr, LogicalPlan};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_deduplicate(
diff --git a/crates/sail-plan/src/resolver/query/filter.rs b/crates/sail-plan/src/resolver/query/filter.rs
index fcb4d7629..d5b030729 100644
--- a/crates/sail-plan/src/resolver/query/filter.rs
+++ b/crates/sail-plan/src/resolver/query/filter.rs
@@ -4,8 +4,8 @@ use datafusion_expr::{Filter, LogicalPlan};
 use sail_common::spec;
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_filter(
diff --git a/crates/sail-plan/src/resolver/query/join.rs b/crates/sail-plan/src/resolver/query/join.rs
index 717f5c20b..ba2f1c628 100644
--- a/crates/sail-plan/src/resolver/query/join.rs
+++ b/crates/sail-plan/src/resolver/query/join.rs
@@ -2,14 +2,14 @@ use std::sync::Arc;
 
 use datafusion_common::{Column, JoinType, NullEquality};
 use datafusion_expr::utils::split_conjunction;
-use datafusion_expr::{build_join_schema, Expr, LogicalPlan, LogicalPlanBuilder};
+use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, build_join_schema};
 use datafusion_functions::expr_fn::coalesce;
 use sail_common::spec;
 use sail_python_udf::udf::pyspark_udf::PySparkUDF;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 /// Returns `true` if the expression is itself a top-level Python scalar UDF call.
 /// This matches Spark SQL's `ExtractPythonUDFFromJoinCondition` optimizer rule
@@ -39,8 +39,7 @@ fn join_type_name(join_type: JoinType) -> &'static str {
     }
 }
 
-const IMPLICIT_CARTESIAN_PRODUCT_MSG: &str =
-    "Detected implicit cartesian product for INNER join between logical plans. \
+const IMPLICIT_CARTESIAN_PRODUCT_MSG: &str = "Detected implicit cartesian product for INNER join between logical plans. \
     Join condition is missing or trivial. \
     Either: use the CROSS JOIN syntax to allow cartesian products between \
     these relations, or: enable implicit cartesian products by setting the \
diff --git a/crates/sail-plan/src/resolver/query/lateral.rs b/crates/sail-plan/src/resolver/query/lateral.rs
index 85208f2c1..f4cb08e37 100644
--- a/crates/sail-plan/src/resolver/query/lateral.rs
+++ b/crates/sail-plan/src/resolver/query/lateral.rs
@@ -9,12 +9,12 @@ use sail_python_udf::udf::pyspark_unresolved_udf::PySparkUnresolvedUDF;
 
 use crate::error::{PlanError, PlanResult};
 use crate::function::get_outer_built_in_generator_functions;
+use crate::resolver::PlanResolver;
 use crate::resolver::function::PythonUdtf;
 use crate::resolver::state::PlanResolverState;
 use crate::resolver::tree::explode::ExplodeRewriter;
 use crate::resolver::tree::monotonic_id::MonotonicIdRewriter;
 use crate::resolver::tree::spark_partition_id::SparkPartitionIdRewriter;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     #[expect(clippy::too_many_arguments)]
@@ -38,10 +38,10 @@ impl PlanResolver<'_> {
         let catalog_manager = self.ctx.extension::()?;
         let mut scope = state.enter_config_scope();
         let state = scope.state();
-        if let Some(f) = catalog_manager.get_function(&canonical_function_name)? {
-            if f.inner().is::() {
-                state.config_mut().arrow_allow_large_var_types = true;
-            }
+        if let Some(f) = catalog_manager.get_function(&canonical_function_name)?
+            && f.inner().is::()
+        {
+            state.config_mut().arrow_allow_large_var_types = true;
         }
         let input = match input {
             Some(x) => self.resolve_query_plan(x, state).await?,
@@ -49,22 +49,24 @@ impl PlanResolver<'_> {
         };
         let schema = input.schema().clone();
 
-        if let Some(f) = catalog_manager.get_function(&canonical_function_name)? {
-            if let Some(f) = f.inner().downcast_ref::() {
-                if !f.eval_type().is_table_function() {
-                    return Err(PlanError::invalid(format!(
-                        "not a table function for UDTF lateral view: {function_name}"
-                    )));
-                }
-                let udtf = PythonUdtf {
-                    python_version: f.python_version().to_string(),
-                    eval_type: f.eval_type(),
-                    command: f.command().to_vec(),
-                    return_type: f.output_type().cloned(),
-                };
-                // Merge named_arguments (SQL kwargs like `col => expr`) into the argument list
-                // as NamedArgument expressions so extract_kwargs can process them uniformly.
-                let all_arguments: Vec = arguments
+        if let Some(f) = catalog_manager.get_function(&canonical_function_name)?
+            && let Some(f) = f.inner().downcast_ref::()
+        {
+            if !f.eval_type().is_table_function() {
+                return Err(PlanError::invalid(format!(
+                    "not a table function for UDTF lateral view: {function_name}"
+                )));
+            }
+            let udtf = PythonUdtf {
+                python_version: f.python_version().to_string(),
+                eval_type: f.eval_type(),
+                command: f.command().to_vec(),
+                return_type: f.output_type().cloned(),
+            };
+            // Merge named_arguments (SQL kwargs like `col => expr`) into the argument list
+            // as NamedArgument expressions so extract_kwargs can process them uniformly.
+            let all_arguments: Vec =
+                arguments
                     .into_iter()
                     .chain(named_arguments.into_iter().map(|(key, value)| {
                         spec::Expr::NamedArgument {
@@ -73,27 +75,26 @@ impl PlanResolver<'_> {
                         }
                     }))
                     .collect();
-                let (positional_args, kwarg_names) = Self::extract_kwargs(all_arguments);
-                let arguments = self
-                    .resolve_named_expressions(positional_args, input.schema(), state)
-                    .await?;
-                let output_names =
-                    column_aliases.map(|aliases| aliases.into_iter().map(|x| x.into()).collect());
-                let output_qualifier = table_alias
-                    .map(|alias| self.resolve_table_reference(&alias))
-                    .transpose()?;
-                return self.resolve_python_udtf_plan(
-                    udtf,
-                    &function_name,
-                    input,
-                    arguments,
-                    &kwarg_names,
-                    output_names,
-                    output_qualifier,
-                    f.deterministic(),
-                    state,
-                );
-            }
+            let (positional_args, kwarg_names) = Self::extract_kwargs(all_arguments);
+            let arguments = self
+                .resolve_named_expressions(positional_args, input.schema(), state)
+                .await?;
+            let output_names =
+                column_aliases.map(|aliases| aliases.into_iter().map(|x| x.into()).collect());
+            let output_qualifier = table_alias
+                .map(|alias| self.resolve_table_reference(&alias))
+                .transpose()?;
+            return self.resolve_python_udtf_plan(
+                udtf,
+                &function_name,
+                input,
+                arguments,
+                &kwarg_names,
+                output_names,
+                output_qualifier,
+                f.deterministic(),
+                state,
+            );
         }
 
         let function_name = if outer {
diff --git a/crates/sail-plan/src/resolver/query/lateral_join.rs b/crates/sail-plan/src/resolver/query/lateral_join.rs
index 6bf77be7d..f78795c11 100644
--- a/crates/sail-plan/src/resolver/query/lateral_join.rs
+++ b/crates/sail-plan/src/resolver/query/lateral_join.rs
@@ -2,16 +2,16 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use datafusion_common::tree_node::{Transformed, TreeNode};
-use datafusion_common::{internal_err, Column, DFSchemaRef, Spans};
+use datafusion_common::{Column, DFSchemaRef, Spans, internal_err};
 use datafusion_expr::expr::Alias;
 use datafusion_expr::{
-    build_join_schema, Expr, LogicalPlan, LogicalPlanBuilder, Projection, Subquery, SubqueryAlias,
+    Expr, LogicalPlan, LogicalPlanBuilder, Projection, Subquery, SubqueryAlias, build_join_schema,
 };
 use sail_common::spec;
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_lateral_join(
diff --git a/crates/sail-plan/src/resolver/query/limit.rs b/crates/sail-plan/src/resolver/query/limit.rs
index 23c586d29..2ed9ee93d 100644
--- a/crates/sail-plan/src/resolver/query/limit.rs
+++ b/crates/sail-plan/src/resolver/query/limit.rs
@@ -3,15 +3,15 @@ use std::sync::Arc;
 use arrow::array::AsArray;
 use arrow::datatypes::Int64Type;
 use datafusion::functions_aggregate::count::count_udaf;
-use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
 use datafusion_common::ScalarValue;
+use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
 use datafusion_expr::expr::AggregateFunctionParams;
-use datafusion_expr::{expr, lit, Aggregate, Expr, Limit, LogicalPlan};
+use datafusion_expr::{Aggregate, Expr, Limit, LogicalPlan, expr, lit};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_limit(
diff --git a/crates/sail-plan/src/resolver/query/misc.rs b/crates/sail-plan/src/resolver/query/misc.rs
index 4dcbb4336..7c60188af 100644
--- a/crates/sail-plan/src/resolver/query/misc.rs
+++ b/crates/sail-plan/src/resolver/query/misc.rs
@@ -14,8 +14,8 @@ use sail_logical_plan::range::RangeNode;
 use sail_logical_plan::repartition::{ExplicitRepartitionKind, ExplicitRepartitionNode};
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     /// Resolves a query plan that produces an empty relation.
diff --git a/crates/sail-plan/src/resolver/query/mod.rs b/crates/sail-plan/src/resolver/query/mod.rs
index 157897fc6..fb117e5bc 100644
--- a/crates/sail-plan/src/resolver/query/mod.rs
+++ b/crates/sail-plan/src/resolver/query/mod.rs
@@ -1,10 +1,16 @@
+use std::sync::Arc;
+
 use async_recursion::async_recursion;
-use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder};
+use datafusion::arrow::datatypes::Schema;
+use datafusion_expr::{Expr, Extension, LogicalPlan, LogicalPlanBuilder};
+use sail_cache::remote_checkpoint::RemoteCheckpointRegistry;
 use sail_common::spec;
+use sail_common_datafusion::extension::SessionExtensionAccessor;
+use sail_logical_plan::remote_checkpoint::RemoteCheckpointRelationNode;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 mod aggregate;
 mod alias;
@@ -227,8 +233,28 @@ impl PlanResolver<'_> {
             QueryNode::CachedLocalRelation { .. } => {
                 return Err(PlanError::todo("cached local relation"));
             }
-            QueryNode::CachedRemoteRelation { .. } => {
-                return Err(PlanError::todo("cached remote relation"));
+            QueryNode::CachedRemoteRelation { relation_id } => {
+                let registry = self.ctx.extension::()?;
+                let descriptor = registry.get(&relation_id)?.ok_or_else(|| {
+                    PlanError::analysis(format!(
+                        "remote checkpoint relation does not exist: {relation_id}"
+                    ))
+                })?;
+                let names = state.register_fields(descriptor.logical_schema.fields());
+                let fields = descriptor
+                    .logical_schema
+                    .fields()
+                    .iter()
+                    .zip(names)
+                    .map(|(field, name)| Arc::new(field.as_ref().clone().with_name(name)))
+                    .collect::>();
+                let schema = Arc::new(Schema::new_with_metadata(
+                    fields,
+                    descriptor.logical_schema.metadata().clone(),
+                ));
+                LogicalPlan::Extension(Extension {
+                    node: Arc::new(RemoteCheckpointRelationNode::try_new(relation_id, schema)?),
+                })
             }
             QueryNode::CommonInlineUserDefinedTableFunction(udtf) => {
                 self.resolve_query_common_inline_udtf(udtf, state).await?
diff --git a/crates/sail-plan/src/resolver/query/na.rs b/crates/sail-plan/src/resolver/query/na.rs
index d56317090..21ad91e4d 100644
--- a/crates/sail-plan/src/resolver/query/na.rs
+++ b/crates/sail-plan/src/resolver/query/na.rs
@@ -3,16 +3,16 @@ use std::sync::Arc;
 use datafusion::arrow::datatypes::DataType;
 use datafusion_expr::utils::conjunction;
 use datafusion_expr::{
-    cast, col, is_false, lit, when, Expr, ExprSchemable, Filter, LogicalPlan, Projection, TryCast,
+    Expr, ExprSchemable, Filter, LogicalPlan, Projection, TryCast, cast, col, is_false, lit, when,
 };
 use datafusion_functions::expr_fn::isnan;
 use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_fill_na(
diff --git a/crates/sail-plan/src/resolver/query/pivoting.rs b/crates/sail-plan/src/resolver/query/pivoting.rs
index fdded7a06..e98baaf2d 100644
--- a/crates/sail-plan/src/resolver/query/pivoting.rs
+++ b/crates/sail-plan/src/resolver/query/pivoting.rs
@@ -2,11 +2,12 @@ use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
 use arrow::datatypes::DataType;
-use datafusion_common::tree_node::{Transformed, TreeNode};
+use datafusion::functions_aggregate::count::count_udaf;
+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
 use datafusion_common::{Column, DFSchema, DFSchemaRef, ScalarValue};
-use datafusion_expr::utils::find_aggregate_exprs;
+use datafusion_expr::expr::{AggregateFunctionParams, NullTreatment};
 use datafusion_expr::{
-    col, expr, lit, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Projection, ScalarUDF,
+    ExprSchemable, LogicalPlan, LogicalPlanBuilder, Projection, ScalarUDF, col, expr, lit, when,
 };
 use datafusion_functions_nested::expr_fn as nested_fn;
 use sail_common::spec;
@@ -16,12 +17,12 @@ use sail_function::scalar::explode;
 use sail_function::scalar::struct_function::StructFunction;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
 use crate::resolver::tree::explode::ExplodeRewriter;
 use crate::resolver::tree::monotonic_id::MonotonicIdRewriter;
 use crate::resolver::tree::spark_partition_id::SparkPartitionIdRewriter;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_pivot(
@@ -40,46 +41,45 @@ impl PlanResolver<'_> {
         let input = self.resolve_query_plan(*input, state).await?;
         let schema = input.schema().clone();
 
-        // Only a single pivot column is supported.
         let mut pivot_columns = self.resolve_expressions(columns, &schema, state).await?;
-        if pivot_columns.len() != 1 {
-            return Err(PlanError::todo("pivot with multiple pivot columns"));
-        }
-        let pivot_column = pivot_columns.swap_remove(0);
+        let column_arity = pivot_columns.len();
+        let pivot_column = match column_arity {
+            0 => {
+                return Err(PlanError::AnalysisError(
+                    "pivot column required".to_string(),
+                ));
+            }
+            1 => pivot_columns.swap_remove(0),
+            _ => make_pivot_struct(pivot_columns),
+        };
 
         let aggregates = self
             .resolve_named_expressions(aggregate, &schema, state)
             .await?;
 
-        // Spark requires every pivot expression to contain an aggregate function. Reject a bare
-        // non-aggregate expression up front with a clear error instead of letting it fail
-        // cryptically (or pass through unfiltered) during the aggregate rewrite.
+        // Spark allows aggregate expressions and pure literals, but rejects columns outside an
+        // aggregate function.
         for agg in &aggregates {
-            if find_aggregate_exprs(std::slice::from_ref(&agg.expr)).is_empty() {
-                return Err(PlanError::AnalysisError(format!(
-                    "Aggregate expression required for pivot, but '{}' did not appear in any \
-                     aggregate function.",
-                    agg.name.join(".")
-                )));
-            }
+            check_valid_pivot_aggregate(&agg.expr, state)?;
         }
 
-        // For SQL pivots the grouping list is empty; the implicit grouping columns are
-        // all input columns not referenced by the pivot column or the aggregates.
-        let grouping = self
-            .resolve_named_expressions(grouping, &schema, state)
-            .await?;
-        let grouping = if grouping.is_empty() {
-            Self::implicit_pivot_grouping(&schema, &pivot_column, &aggregates, state)?
-        } else {
-            grouping
+        // For SQL pivots the grouping list is absent; the implicit grouping columns are
+        // all input columns not referenced by the pivot column or the aggregates. The
+        // DataFrame API always sets the list, and an empty one means grouping by nothing.
+        let grouping = match grouping {
+            Some(grouping) => {
+                self.resolve_named_expressions(grouping, &schema, state)
+                    .await?
+            }
+            None => Self::implicit_pivot_grouping(&schema, &pivot_column, &aggregates, state)?,
         };
 
         // Each pivot value becomes one output column per aggregate. Values may be
         // given explicitly (`... IN (v1, v2)`); otherwise infer the distinct values
         // from the data (matching Spark's `pivot(column)`).
         let pivot_values: Vec<(ScalarValue, Option)> = if values.is_empty() {
-            self.infer_pivot_values(&input, &pivot_column)
+            let pivot_column_name = pivot_column_display_name(&pivot_column, state)?;
+            self.infer_pivot_values(&input, &pivot_column, &pivot_column_name)
                 .await?
                 .into_iter()
                 .map(|scalar| (scalar, None))
@@ -96,17 +96,21 @@ impl PlanResolver<'_> {
                     values: exprs,
                     alias,
                 } = value;
-                if exprs.len() != 1 {
-                    return Err(PlanError::todo(
-                        "pivot value with multiple expressions (struct pivot)",
-                    ));
+                if exprs.len() != column_arity {
+                    return Err(PlanError::AnalysisError(format!(
+                        "pivot value has {} expressions, but pivot column has {}",
+                        exprs.len(),
+                        column_arity
+                    )));
                 }
-                let Some(expr) = exprs.into_iter().next() else {
-                    return Err(PlanError::AnalysisError(
-                        "pivot value must have a value".to_string(),
-                    ));
+                let mut exprs = self
+                    .resolve_expressions(exprs, &empty_schema, state)
+                    .await?;
+                let expr = if column_arity == 1 {
+                    exprs.swap_remove(0)
+                } else {
+                    make_pivot_struct(exprs)
                 };
-                let expr = self.resolve_expression(expr, &empty_schema, state).await?;
                 let scalar = evaluator
                     .evaluate(&expr)
                     .map_err(|e| PlanError::invalid(e.to_string()))?;
@@ -115,6 +119,19 @@ impl PlanResolver<'_> {
             resolved
         };
 
+        // Spark casts each pivot value to the pivot column's type before comparing
+        // (`Cast(value, pivotColumn.dataType)`), so an `int` literal matches a `bigint` column and
+        // a struct value matches the struct column's field types. The output name still uses the
+        // raw value (Spark casts the original value to string), so it is taken from `scalar` below
+        // before the cast.
+        let column_type = pivot_column.get_type(&schema)?;
+        let spark_uses_pivot_first = aggregates
+            .iter()
+            .map(|agg| agg.expr.get_type(&schema))
+            .collect::, _>>()?
+            .iter()
+            .all(spark_pivot_first_supports_data_type);
+        let force_first_last_ignore_nulls = !spark_uses_pivot_first;
         let single_aggregate = aggregates.len() == 1;
         let mut projections = grouping.clone();
         for (scalar, alias) in pivot_values {
@@ -130,10 +147,41 @@ impl PlanResolver<'_> {
             let predicate = if is_null {
                 pivot_column.clone().is_null()
             } else {
-                pivot_column.clone().eq(lit(scalar))
+                let scalar = scalar
+                    .cast_to(&column_type)
+                    .map_err(|e| PlanError::invalid(e.to_string()))?;
+                // Spark's PivotFirst fast path creates a NaN column but does not match NaN rows into it.
+                if spark_uses_pivot_first && scalar_is_nan(&scalar) {
+                    lit(false)
+                } else {
+                    pivot_column.clone().eq(lit(scalar))
+                }
+            };
+            // On the PivotFirst path, an absent (group, pivot-value) combination is NULL even
+            // for `count` (Spark only computes the aggregate over (group, value) pairs that
+            // occur in the data). The aggregate FILTER below makes `count` return 0 for such a
+            // group, so gate each cell on whether any row matched the pivot value. This is a
+            // no-op for sum/avg/min/max (already NULL on empty) and corrects the counting
+            // family (count, count(DISTINCT), approx_count_distinct). On the general path
+            // (`spark_uses_pivot_first == false`) Spark itself returns 0 there, matching the
+            // unguarded FILTER, so no guard is applied.
+            let row_present = if spark_uses_pivot_first {
+                Some(count_rows_matching(&predicate).gt(lit(0i64)))
+            } else {
+                None
             };
             for agg in &aggregates {
-                let expr = inject_pivot_filter(agg.expr.clone(), &predicate)?;
+                let expr = inject_pivot_filter(
+                    agg.expr.clone(),
+                    &predicate,
+                    force_first_last_ignore_nulls,
+                )?;
+                let expr = match &row_present {
+                    Some(condition) => {
+                        when(condition.clone(), expr).otherwise(lit(ScalarValue::Null))?
+                    }
+                    None => expr,
+                };
                 let name = if single_aggregate {
                     value_name.clone()
                 } else {
@@ -156,6 +204,7 @@ impl PlanResolver<'_> {
         &self,
         input: &LogicalPlan,
         pivot_column: &expr::Expr,
+        pivot_column_name: &str,
     ) -> PlanResult> {
         // Spark rejects pivots whose distinct-value count exceeds the configured maximum
         // (`spark.sql.pivotMaxValues`, default 10000), defined as `DATAFRAME_PIVOT_MAX_VALUES`
@@ -170,7 +219,7 @@ impl PlanResolver<'_> {
         // values into the planner process; the extra row still lets us detect overflow.
         let plan = LogicalPlanBuilder::from(input.clone())
             .aggregate(vec![pivot_column.clone()], Vec::::new())?
-            .limit(0, Some(max_pivot_values + 1))?
+            .limit(0, Some(max_pivot_values.saturating_add(1)))?
             .build()?;
         let batches = self.ctx.execute_logical_plan(plan).await?.collect().await?;
         let mut values = Vec::new();
@@ -184,8 +233,10 @@ impl PlanResolver<'_> {
         // an exact count (which would be capped and misleading) — mirroring Spark's message.
         if values.len() > max_pivot_values {
             return Err(PlanError::AnalysisError(format!(
-                "The pivot column has more than {max_pivot_values} distinct values; this could \
-                 indicate an error. Specify the pivot values explicitly if this was intended."
+                "The pivot column {pivot_column_name} has more than {max_pivot_values} distinct \
+                 values, this could indicate an error. If this was intended, set \
+                 spark.sql.pivotMaxValues to at least the number of distinct values of the pivot \
+                 column."
             )));
         }
         // `partial_cmp` returns `None` only for incomparable values such as float `NaN`.
@@ -404,10 +455,102 @@ fn pivot_value_name(scalar: &ScalarValue) -> PlanResult {
         .map_err(|e| PlanError::invalid(e.to_string()))
 }
 
+fn make_pivot_struct(exprs: Vec) -> expr::Expr {
+    ScalarUDF::from(StructFunction::new(
+        (0..exprs.len()).map(|i| format!("col{}", i + 1)).collect(),
+    ))
+    .call(exprs)
+}
+
+fn pivot_column_display_name(expr: &expr::Expr, state: &PlanResolverState) -> PlanResult {
+    if let expr::Expr::Column(column) = expr {
+        Ok(state.get_field_info(column.name())?.name().to_string())
+    } else {
+        Ok(expr.to_string())
+    }
+}
+
+fn check_valid_pivot_aggregate(expr: &expr::Expr, state: &PlanResolverState) -> PlanResult<()> {
+    let mut bare_column = None;
+
+    expr.apply(|e| match e {
+        expr::Expr::AggregateFunction(_) => Ok(TreeNodeRecursion::Jump),
+        expr::Expr::Column(column) => {
+            bare_column = Some(column.clone());
+            Ok(TreeNodeRecursion::Stop)
+        }
+        _ => Ok(TreeNodeRecursion::Continue),
+    })?;
+
+    if let Some(column) = bare_column {
+        let name = state
+            .get_field_info(column.name())
+            .map(|info| info.name().to_string())
+            .unwrap_or_else(|_| column.name().to_string());
+        return Err(PlanError::AnalysisError(format!(
+            "Aggregate expression required for pivot, but '{name}' did not appear in any \
+             aggregate function."
+        )));
+    }
+
+    Ok(())
+}
+
+fn spark_pivot_first_supports_data_type(data_type: &DataType) -> bool {
+    matches!(
+        data_type,
+        DataType::Boolean
+            | DataType::Int8
+            | DataType::Int16
+            | DataType::Int32
+            | DataType::Int64
+            | DataType::UInt8
+            | DataType::UInt16
+            | DataType::UInt32
+            | DataType::UInt64
+            | DataType::Float16
+            | DataType::Float32
+            | DataType::Float64
+            | DataType::Decimal32(_, _)
+            | DataType::Decimal64(_, _)
+            | DataType::Decimal128(_, _)
+            | DataType::Decimal256(_, _)
+    )
+}
+
+fn scalar_is_nan(scalar: &ScalarValue) -> bool {
+    match scalar {
+        ScalarValue::Float16(Some(value)) => value.is_nan(),
+        ScalarValue::Float32(Some(value)) => value.is_nan(),
+        ScalarValue::Float64(Some(value)) => value.is_nan(),
+        _ => false,
+    }
+}
+
+/// Build `count(1) FILTER (WHERE predicate)`: the number of rows in the group that match
+/// the pivot value. Used to distinguish an absent (group, pivot-value) combination (zero
+/// matching rows -> NULL cell) from a present one whose aggregate happens to be `0`.
+fn count_rows_matching(predicate: &expr::Expr) -> expr::Expr {
+    expr::Expr::AggregateFunction(expr::AggregateFunction {
+        func: count_udaf(),
+        params: AggregateFunctionParams {
+            args: vec![lit(1i64)],
+            distinct: false,
+            filter: Some(Box::new(predicate.clone())),
+            order_by: vec![],
+            null_treatment: None,
+        },
+    })
+}
+
 /// Add `predicate` as a FILTER to every aggregate function inside `expr`,
 /// AND-combining with any existing filter. Implements Spark's pivot semantics:
 /// each aggregate is computed only over rows matching the pivot value.
-fn inject_pivot_filter(expr: expr::Expr, predicate: &expr::Expr) -> PlanResult {
+fn inject_pivot_filter(
+    expr: expr::Expr,
+    predicate: &expr::Expr,
+    force_first_last_ignore_nulls: bool,
+) -> PlanResult {
     Ok(expr
         .transform(|e| match e {
             expr::Expr::AggregateFunction(mut func) => {
@@ -416,6 +559,11 @@ fn inject_pivot_filter(expr: expr::Expr, predicate: &expr::Expr) -> PlanResult predicate.clone(),
                 };
                 func.params.filter = Some(Box::new(filter));
+                if force_first_last_ignore_nulls
+                    && matches!(func.func.name(), "first_value" | "last_value")
+                {
+                    func.params.null_treatment = Some(NullTreatment::IgnoreNulls);
+                }
                 Ok(Transformed::yes(expr::Expr::AggregateFunction(func)))
             }
             other => Ok(Transformed::no(other)),
diff --git a/crates/sail-plan/src/resolver/query/project.rs b/crates/sail-plan/src/resolver/query/project.rs
index ffcd9faba..774b68e58 100644
--- a/crates/sail-plan/src/resolver/query/project.rs
+++ b/crates/sail-plan/src/resolver/query/project.rs
@@ -12,14 +12,14 @@ use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::multi_expr::MultiExpr;
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
+use crate::resolver::tree::PlanRewriter;
 use crate::resolver::tree::explode::ExplodeRewriter;
 use crate::resolver::tree::monotonic_id::MonotonicIdRewriter;
 use crate::resolver::tree::spark_partition_id::SparkPartitionIdRewriter;
 use crate::resolver::tree::window::WindowRewriter;
-use crate::resolver::tree::PlanRewriter;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_project(
@@ -103,10 +103,10 @@ impl PlanResolver<'_> {
 
             if let Some(outer_schema) = state.get_outer_query_schema() {
                 for (qualifier, field) in outer_schema.iter() {
-                    if let Some(expected) = qualifier_filter {
-                        if qualifier.is_none_or(|q| expected != q) {
-                            continue;
-                        }
+                    if let Some(expected) = qualifier_filter
+                        && qualifier.is_none_or(|q| expected != q)
+                    {
+                        continue;
                     }
                     let info = state.get_field_info(field.name())?;
                     if info.is_hidden() {
diff --git a/crates/sail-plan/src/resolver/query/read.rs b/crates/sail-plan/src/resolver/query/read.rs
index 37ce9811a..ff6781990 100644
--- a/crates/sail-plan/src/resolver/query/read.rs
+++ b/crates/sail-plan/src/resolver/query/read.rs
@@ -3,10 +3,10 @@ use std::sync::Arc;
 
 use datafusion::arrow::datatypes::{DataType, Schema};
 use datafusion::catalog::TableFunctionArgs;
-use datafusion::datasource::{provider_as_source, source_as_provider, TableProvider};
+use datafusion::datasource::{TableProvider, provider_as_source, source_as_provider};
 use datafusion_common::{DFSchema, ScalarValue, TableReference};
 use datafusion_expr::{Expr, LogicalPlan, SubqueryAlias, TableScan, TableSource, UNNAMED_TABLE};
-use rand::{rng, RngExt};
+use rand::{RngExt, rng};
 use sail_catalog::manager::CatalogManager;
 use sail_common::spec;
 use sail_common_datafusion::catalog::{LakehouseOperation, TableColumnStatus, TableKind};
@@ -20,9 +20,9 @@ use sail_python_udf::udf::pyspark_unresolved_udf::PySparkUnresolvedUDF;
 
 use crate::error::{PlanError, PlanResult};
 use crate::function::{get_built_in_table_function, is_built_in_generator_function};
+use crate::resolver::PlanResolver;
 use crate::resolver::function::PythonUdtf;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     /// Resolves a named table or view reference into a logical plan node.
@@ -417,16 +417,17 @@ impl PlanResolver<'_> {
             } else {
                 let schema = Arc::new(DFSchema::empty());
                 let arguments = self.resolve_expressions(arguments, &schema, state).await?;
-                let table_function =
-                    if let Ok(f) = self.ctx.table_function(&canonical_function_name) {
-                        f
-                    } else if let Ok(f) = get_built_in_table_function(&canonical_function_name) {
-                        f
-                    } else {
-                        return Err(PlanError::unsupported(format!(
-                            "unknown table function: {function_name}"
-                        )));
-                    };
+                let table_function = match self.ctx.table_function(&canonical_function_name) {
+                    Ok(f) => f,
+                    _ => match get_built_in_table_function(&canonical_function_name) {
+                        Ok(f) => f,
+                        _ => {
+                            return Err(PlanError::unsupported(format!(
+                                "unknown table function: {function_name}"
+                            )));
+                        }
+                    },
+                };
                 let session_state = self.ctx.state();
                 let table_provider = table_function.create_table_provider_with_args(
                     TableFunctionArgs::new(&arguments, &session_state),
diff --git a/crates/sail-plan/src/resolver/query/recursion.rs b/crates/sail-plan/src/resolver/query/recursion.rs
index 2543cad1b..6ed7a61b8 100644
--- a/crates/sail-plan/src/resolver/query/recursion.rs
+++ b/crates/sail-plan/src/resolver/query/recursion.rs
@@ -2,8 +2,8 @@ use datafusion_expr::LogicalPlan;
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_recursive_query_plan(
diff --git a/crates/sail-plan/src/resolver/query/repartition.rs b/crates/sail-plan/src/resolver/query/repartition.rs
index b34d5b1ee..523a66293 100644
--- a/crates/sail-plan/src/resolver/query/repartition.rs
+++ b/crates/sail-plan/src/resolver/query/repartition.rs
@@ -5,8 +5,8 @@ use sail_common::spec;
 use sail_logical_plan::repartition::{ExplicitRepartitionKind, ExplicitRepartitionNode};
 
 use crate::error::PlanResult;
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_repartition(
diff --git a/crates/sail-plan/src/resolver/query/sample.rs b/crates/sail-plan/src/resolver/query/sample.rs
index 0adff1f88..76694508e 100644
--- a/crates/sail-plan/src/resolver/query/sample.rs
+++ b/crates/sail-plan/src/resolver/query/sample.rs
@@ -4,8 +4,8 @@ use datafusion::arrow::datatypes::DataType;
 use datafusion_common::ScalarValue;
 use datafusion_expr::expr::ScalarFunction;
 use datafusion_expr::select_expr::SelectExpr;
-use datafusion_expr::{col, lit, Expr, Extension, LogicalPlan, LogicalPlanBuilder, ScalarUDF};
-use rand::{rng, RngExt};
+use datafusion_expr::{Expr, Extension, LogicalPlan, LogicalPlanBuilder, ScalarUDF, col, lit};
+use rand::{RngExt, rng};
 use sail_common::spec;
 use sail_common::spec::{NullOrdering, SortDirection, SortOrder};
 use sail_function::scalar::array::spark_sequence::SparkSequence;
@@ -14,8 +14,8 @@ use sail_function::scalar::math::random::Random;
 use sail_logical_plan::sort::SortWithinPartitionsNode;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 /// Copied from `arrow_ord::rank::can_rank` (private in arrow-ord).
 fn can_rank(data_type: &DataType) -> bool {
diff --git a/crates/sail-plan/src/resolver/query/set_op.rs b/crates/sail-plan/src/resolver/query/set_op.rs
index f7090bc0d..1c0f2320e 100644
--- a/crates/sail-plan/src/resolver/query/set_op.rs
+++ b/crates/sail-plan/src/resolver/query/set_op.rs
@@ -4,13 +4,13 @@ use datafusion_common::{Column, JoinType, NullEquality, ScalarValue};
 use datafusion_expr::builder::project;
 use datafusion_expr::expr::WindowFunctionParams;
 use datafusion_expr::{
-    expr, Expr, LogicalPlan, LogicalPlanBuilder, WindowFrame, WindowFunctionDefinition,
+    Expr, LogicalPlan, LogicalPlanBuilder, WindowFrame, WindowFunctionDefinition, expr,
 };
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_set_operation(
diff --git a/crates/sail-plan/src/resolver/query/sort.rs b/crates/sail-plan/src/resolver/query/sort.rs
index 144c16964..65c31ba11 100644
--- a/crates/sail-plan/src/resolver/query/sort.rs
+++ b/crates/sail-plan/src/resolver/query/sort.rs
@@ -1,8 +1,8 @@
 use std::sync::Arc;
 
 use async_recursion::async_recursion;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
 use datafusion_common::Column;
+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
 use datafusion_expr::expr::{Alias, Sort};
 use datafusion_expr::{
     Aggregate, Expr, Extension, LogicalPlan, LogicalPlanBuilder, Projection, Window,
@@ -12,8 +12,8 @@ use sail_common_datafusion::utils::items::ItemTaker;
 use sail_logical_plan::sort::SortWithinPartitionsNode;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_sort(
@@ -136,20 +136,20 @@ impl PlanResolver<'_> {
                     name,
                     ..
                 }) = expr
+                    && relation == &col.relation
+                    && name == &col.name
                 {
-                    if relation == &col.relation && name == &col.name {
-                        return Some(expr.as_ref().clone());
-                    }
+                    return Some(expr.as_ref().clone());
                 }
                 None
             })
         };
         let expr = expr
             .transform_down(|e| {
-                if let Expr::Column(ref col) = e {
-                    if let Some(expr) = find(col) {
-                        return Ok(Transformed::yes(expr));
-                    }
+                if let Expr::Column(ref col) = e
+                    && let Some(expr) = find(col)
+                {
+                    return Ok(Transformed::yes(expr));
                 }
                 Ok(Transformed::no(e))
             })
@@ -175,10 +175,9 @@ impl PlanResolver<'_> {
                     name,
                     ..
                 }) = expr
+                    && expr.as_ref() == target
                 {
-                    if expr.as_ref() == target {
-                        return Some(Expr::Column(Column::new(relation.clone(), name.clone())));
-                    }
+                    return Some(Expr::Column(Column::new(relation.clone(), name.clone())));
                 }
                 None
             })
diff --git a/crates/sail-plan/src/resolver/query/stat.rs b/crates/sail-plan/src/resolver/query/stat.rs
index 642c92765..639a5d3d4 100644
--- a/crates/sail-plan/src/resolver/query/stat.rs
+++ b/crates/sail-plan/src/resolver/query/stat.rs
@@ -13,15 +13,15 @@ use datafusion::functions_aggregate::sum::sum_udaf;
 use datafusion_common::{Column, ExprSchema, ScalarValue};
 use datafusion_expr::expr::{AggregateFunctionParams, ScalarFunction};
 use datafusion_expr::{
-    and, col, expr, lit, or, Expr, ExprSchemable, LogicalPlan, LogicalPlanBuilder, ScalarUDF,
+    Expr, ExprSchemable, LogicalPlan, LogicalPlanBuilder, ScalarUDF, and, col, expr, lit, or,
 };
 use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::math::random::Random;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_stat_summary(
@@ -218,11 +218,10 @@ impl PlanResolver<'_> {
         let create_stat_row =
             |stat_name: &str, stats_by_column: Vec<(String, Expr)>| -> PlanResult {
                 let stats_plan_clone = stats_plan.clone();
-                let mut projections =
-                    vec![
-                        Expr::Literal(ScalarValue::Utf8(Some(stat_name.to_string())), None)
-                            .alias(&summary_alias),
-                    ];
+                let mut projections = vec![
+                    Expr::Literal(ScalarValue::Utf8(Some(stat_name.to_string())), None)
+                        .alias(&summary_alias),
+                ];
                 for (col_name, expr) in stats_by_column {
                     let expr = expr.cast_to(&DataType::Utf8, stats_plan_clone.schema())?;
                     projections.push(expr.alias(&col_name));
@@ -339,14 +338,13 @@ impl PlanResolver<'_> {
         let right_column = self.resolve_one_column(input.schema(), right_column.as_ref(), state)?;
 
         let projected_plan = LogicalPlanBuilder::from(input.clone())
-            .project(vec![Expr::Cast(expr::Cast::new(
-                Box::new(col(right_column.clone())),
-                DataType::Utf8,
-            ))
-            .alias_qualified(
-                right_column.relation.clone(),
-                right_column.name.clone(),
-            )])?
+            .project(vec![
+                Expr::Cast(expr::Cast::new(
+                    Box::new(col(right_column.clone())),
+                    DataType::Utf8,
+                ))
+                .alias_qualified(right_column.relation.clone(), right_column.name.clone()),
+            ])?
             .build()?;
         let distinct_values = LogicalPlanBuilder::from(projected_plan)
             .project(vec![Expr::Column(right_column.clone())])?
diff --git a/crates/sail-plan/src/resolver/query/time_travel.rs b/crates/sail-plan/src/resolver/query/time_travel.rs
index 65887a5c9..f9de62f64 100644
--- a/crates/sail-plan/src/resolver/query/time_travel.rs
+++ b/crates/sail-plan/src/resolver/query/time_travel.rs
@@ -8,8 +8,8 @@ use datafusion_expr::{EmptyRelation, Expr, Limit, LogicalPlan, Projection};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_time_travel_options(
diff --git a/crates/sail-plan/src/resolver/query/udf.rs b/crates/sail-plan/src/resolver/query/udf.rs
index 34ed42b0f..579a4e706 100644
--- a/crates/sail-plan/src/resolver/query/udf.rs
+++ b/crates/sail-plan/src/resolver/query/udf.rs
@@ -4,8 +4,8 @@ use datafusion::arrow::datatypes::{DataType, Field, Schema};
 use datafusion_common::{Column, DFSchema, JoinType};
 use datafusion_expr::expr::{AggregateFunctionParams, ScalarFunction};
 use datafusion_expr::{
-    expr, ident, AggregateUDF, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder,
-    ScalarUDF,
+    AggregateUDF, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, ScalarUDF, expr,
+    ident,
 };
 use datafusion_functions::core::expr_ext::FieldAccessor;
 use sail_common::spec;
@@ -19,9 +19,9 @@ use sail_python_udf::udf::pyspark_group_map_udf::{PySparkGroupMapMode, PySparkGr
 use sail_python_udf::udf::pyspark_map_iter_udf::{PySparkMapIterKind, PySparkMapIterUDF};
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_map_partitions(
@@ -55,7 +55,7 @@ impl PlanResolver<'_> {
             _ => {
                 return Err(PlanError::invalid(
                     "MapPartitions UDF output type must be struct",
-                ))
+                ));
             }
         };
         let output_names = state.register_fields(output_schema.fields());
@@ -173,7 +173,7 @@ impl PlanResolver<'_> {
             _ => {
                 return Err(PlanError::invalid(
                     "GroupMap UDF output type must be struct",
-                ))
+                ));
             }
         };
         let udf_output_type = DataType::List(Arc::new(Field::new_list_field(
@@ -327,7 +327,7 @@ impl PlanResolver<'_> {
             _ => {
                 return Err(PlanError::invalid(
                     "GroupMap UDF output type must be struct",
-                ))
+                ));
             }
         };
         let mapper_output_type = DataType::List(Arc::new(Field::new_list_field(
diff --git a/crates/sail-plan/src/resolver/query/udtf.rs b/crates/sail-plan/src/resolver/query/udtf.rs
index 40497411f..ef3b77f32 100644
--- a/crates/sail-plan/src/resolver/query/udtf.rs
+++ b/crates/sail-plan/src/resolver/query/udtf.rs
@@ -15,11 +15,11 @@ use sail_python_udf::get_udf_name;
 use sail_python_udf::udf::pyspark_udtf::{PySparkUDTF, PySparkUdtfKind};
 
 use crate::error::{PlanError, PlanResult};
+use crate::resolver::PlanResolver;
 use crate::resolver::expression::NamedExpr;
 use crate::resolver::function::PythonUdtf;
 use crate::resolver::state::PlanResolverState;
 use crate::resolver::tree::table_input::TableInputRewriter;
-use crate::resolver::PlanResolver;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_common_inline_udtf(
@@ -87,6 +87,19 @@ impl PlanResolver<'_> {
             })
             .collect::>>()?;
 
+        let table_argument_count = argument_is_tables
+            .iter()
+            .filter(|is_table| **is_table)
+            .count();
+        if table_argument_count > 1 && !self.config.tvf_allow_multiple_table_arguments {
+            return Err(PlanError::analysis(format!(
+                "[TABLE_VALUED_FUNCTION_TOO_MANY_TABLE_ARGUMENTS] There are too many table \
+                 arguments for table-valued function. It allows one table argument, but got: \
+                 {table_argument_count}. If you want to allow it, please set \
+                 \"spark.sql.tvf.allowMultipleTableArguments.enabled\" to \"true\""
+            )));
+        }
+
         let kind = match function.eval_type {
             spec::PySparkUdfType::Table => PySparkUdtfKind::Table,
             spec::PySparkUdfType::ArrowTable | spec::PySparkUdfType::ArrowUdtf => {
@@ -96,7 +109,7 @@ impl PlanResolver<'_> {
                 return Err(PlanError::invalid(format!(
                     "PySpark UDTF type: {:?}",
                     function.eval_type,
-                )))
+                )));
             }
         };
         // Determine the number of passthrough columns before rewriting the plan
@@ -158,6 +171,7 @@ impl PlanResolver<'_> {
                     &arg_literals,
                     kwargs,
                     &argument_is_tables,
+                    self.config.pyspark_udf_config.arrow_use_large_var_types,
                 )?
             }
         };
@@ -168,6 +182,7 @@ impl PlanResolver<'_> {
             function.eval_type,
             arguments_len,
             &input_types,
+            passthrough_columns,
             kwargs,
             &return_type,
             &self.config.pyspark_udf_config,
diff --git a/crates/sail-plan/src/resolver/query/values.rs b/crates/sail-plan/src/resolver/query/values.rs
index 571a34d36..1fd7328a1 100644
--- a/crates/sail-plan/src/resolver/query/values.rs
+++ b/crates/sail-plan/src/resolver/query/values.rs
@@ -3,12 +3,12 @@ use std::sync::Arc;
 
 use datafusion::arrow::datatypes::DataType;
 use datafusion_common::{DFSchema, DFSchemaRef};
-use datafusion_expr::{cast, Expr, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Projection};
+use datafusion_expr::{Expr, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Projection, cast};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_query_values(
@@ -51,14 +51,14 @@ impl PlanResolver<'_> {
         let mut nan_positions = HashSet::new();
         for value in values.iter() {
             value.iter().enumerate().for_each(|(idx, expr)| {
-                if let Expr::Cast(cast) = expr {
-                    if let Expr::Literal(sv, _) = cast.expr.as_ref() {
-                        if let Some(true) = sv.try_as_str().flatten().map(|s| {
-                            s.to_uppercase() == "NAN" && cast.field.data_type().is_numeric()
-                        }) {
-                            nan_positions.insert(idx);
-                        }
-                    }
+                if let Expr::Cast(cast) = expr
+                    && let Expr::Literal(sv, _) = cast.expr.as_ref()
+                    && let Some(true) = sv
+                        .try_as_str()
+                        .flatten()
+                        .map(|s| s.to_uppercase() == "NAN" && cast.field.data_type().is_numeric())
+                {
+                    nan_positions.insert(idx);
                 }
             });
         }
diff --git a/crates/sail-plan/src/resolver/query/window.rs b/crates/sail-plan/src/resolver/query/window.rs
index 0f0e2ae09..ddba6e56d 100644
--- a/crates/sail-plan/src/resolver/query/window.rs
+++ b/crates/sail-plan/src/resolver/query/window.rs
@@ -4,8 +4,8 @@ use datafusion_expr::LogicalPlan;
 use sail_common::spec::{self, Identifier};
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) async fn resolve_named_windows(
diff --git a/crates/sail-plan/src/resolver/query/with_relations.rs b/crates/sail-plan/src/resolver/query/with_relations.rs
index 61ec374e8..70d55b2cd 100644
--- a/crates/sail-plan/src/resolver/query/with_relations.rs
+++ b/crates/sail-plan/src/resolver/query/with_relations.rs
@@ -4,8 +4,8 @@ use datafusion_expr::{LogicalPlan, SubqueryAlias};
 use sail_common::spec;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     /// Resolves a WithRelations node by storing references in state, then resolving the root.
diff --git a/crates/sail-plan/src/resolver/schema.rs b/crates/sail-plan/src/resolver/schema.rs
index be98293cd..8fac7af94 100644
--- a/crates/sail-plan/src/resolver/schema.rs
+++ b/crates/sail-plan/src/resolver/schema.rs
@@ -5,8 +5,8 @@ use sail_common::spec;
 use sail_common_datafusion::utils::items::ItemTaker;
 
 use crate::error::{PlanError, PlanResult};
-use crate::resolver::state::PlanResolverState;
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
 
 impl PlanResolver<'_> {
     pub(super) fn resolve_table_reference(
diff --git a/crates/sail-plan/src/resolver/tree/explode.rs b/crates/sail-plan/src/resolver/tree/explode.rs
index c766076ee..cd9c2b881 100644
--- a/crates/sail-plan/src/resolver/tree/explode.rs
+++ b/crates/sail-plan/src/resolver/tree/explode.rs
@@ -7,19 +7,20 @@ use datafusion::common::{Column, Result, UnnestOptions};
 use datafusion::functions::core::expr_ext::FieldAccessor;
 use datafusion::logical_expr::builder::unnest_with_options;
 use datafusion::logical_expr::{Expr, ExprSchemable, LogicalPlan, Projection, ScalarUDF};
-use datafusion_common::{plan_err, ExprSchema};
+use datafusion_common::{ExprSchema, plan_err};
 use datafusion_expr::expr::ScalarFunction;
-use datafusion_expr::{ident, when};
+use datafusion_expr::{ident, lit, when};
 use datafusion_functions_nested::expr_fn as nested_fn;
 use either::Either;
 use sail_common::spec::{SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::array::spark_array_item_with_position::ArrayItemWithPosition;
+use sail_function::scalar::array_struct_field::ArrayStructField;
 use sail_function::scalar::explode::{Explode, ExplodeKind};
 use sail_function::scalar::multi_expr::MultiExpr;
 
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::tree::{empty_logical_plan, PlanRewriter};
+use crate::resolver::tree::{PlanRewriter, empty_logical_plan};
 
 enum ExplodeDataType {
     List,
@@ -94,6 +95,7 @@ impl TreeNodeRewriter for ExplodeRewriter<'_> {
         };
 
         let name = self.state.register_field_name("");
+        let mut inline_projections = vec![];
         let out = match (data_type, with_position, is_inline) {
             (ExplodeDataType::List, false, false) => vec![ident(&name).alias("col")],
             (ExplodeDataType::List, true, false) => {
@@ -106,9 +108,15 @@ impl TreeNodeRewriter for ExplodeRewriter<'_> {
                 DataType::Struct(fields) => Ok(fields
                     .into_iter()
                     .map(|field| {
-                        ident(&name)
-                            .field(field.name().as_str())
-                            .alias(field.name().as_str())
+                        let field_name = field.name().to_string();
+                        let field_column = self.state.register_field_name("");
+                        inline_projections.push((
+                            ScalarUDF::from(ArrayStructField::new())
+                                .call(vec![arg.clone(), lit(field_name.clone())])
+                                .alias(&field_column),
+                            Column::from_name(&field_column),
+                        ));
+                        ident(&field_column).alias(field_name)
                     })
                     .collect::>()),
                 wrong_type => plan_err!(
@@ -141,7 +149,17 @@ impl TreeNodeRewriter for ExplodeRewriter<'_> {
             .into_iter()
             .map(Expr::Column)
             .collect::>();
-        projections.push(arg.alias(&name));
+        let columns_to_unnest = if inline_projections.is_empty() {
+            projections.push(arg.alias(&name));
+            vec![Column::from_name(&name)]
+        } else {
+            let columns = inline_projections
+                .iter()
+                .map(|(_, column)| column.clone())
+                .collect::>();
+            projections.extend(inline_projections.into_iter().map(|(expr, _)| expr));
+            columns
+        };
 
         let plan = mem::replace(&mut self.plan, empty_logical_plan());
         // TODO: If specific columns need to be unnested multiple times (e. g at different depth), declare them here.
@@ -149,7 +167,7 @@ impl TreeNodeRewriter for ExplodeRewriter<'_> {
         let recursions = vec![];
         self.plan = unnest_with_options(
             LogicalPlan::Projection(Projection::try_new(projections, Arc::new(plan))?),
-            vec![Column::from_name(&name)],
+            columns_to_unnest,
             UnnestOptions {
                 preserve_nulls,
                 recursions,
diff --git a/crates/sail-plan/src/resolver/tree/monotonic_id.rs b/crates/sail-plan/src/resolver/tree/monotonic_id.rs
index 74b59a0e2..453af42ed 100644
--- a/crates/sail-plan/src/resolver/tree/monotonic_id.rs
+++ b/crates/sail-plan/src/resolver/tree/monotonic_id.rs
@@ -1,16 +1,16 @@
 use std::mem;
 use std::sync::Arc;
 
-use datafusion::common::tree_node::{Transformed, TreeNodeRewriter};
 use datafusion::common::Result;
+use datafusion::common::tree_node::{Transformed, TreeNodeRewriter};
 use datafusion_expr::expr::ScalarFunction;
-use datafusion_expr::{ident, Expr, Extension, LogicalPlan};
+use datafusion_expr::{Expr, Extension, LogicalPlan, ident};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::misc::monotonically_increasing_id::SparkMonotonicallyIncreasingId;
 use sail_logical_plan::monotonic_id::MonotonicIdNode;
 
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::tree::{empty_logical_plan, PlanRewriter};
+use crate::resolver::tree::{PlanRewriter, empty_logical_plan};
 
 pub(crate) struct MonotonicIdRewriter<'s> {
     plan: LogicalPlan,
diff --git a/crates/sail-plan/src/resolver/tree/spark_partition_id.rs b/crates/sail-plan/src/resolver/tree/spark_partition_id.rs
index 0da367d57..ead3d3d76 100644
--- a/crates/sail-plan/src/resolver/tree/spark_partition_id.rs
+++ b/crates/sail-plan/src/resolver/tree/spark_partition_id.rs
@@ -1,16 +1,16 @@
 use std::mem;
 use std::sync::Arc;
 
-use datafusion::common::tree_node::{Transformed, TreeNodeRewriter};
 use datafusion::common::Result;
+use datafusion::common::tree_node::{Transformed, TreeNodeRewriter};
 use datafusion_expr::expr::ScalarFunction;
-use datafusion_expr::{ident, Expr, Extension, LogicalPlan};
+use datafusion_expr::{Expr, Extension, LogicalPlan, ident};
 use sail_common_datafusion::utils::items::ItemTaker;
 use sail_function::scalar::misc::spark_partition_id::SparkPartitionId;
 use sail_logical_plan::spark_partition_id::SparkPartitionIdNode;
 
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::tree::{empty_logical_plan, PlanRewriter};
+use crate::resolver::tree::{PlanRewriter, empty_logical_plan};
 
 pub(crate) struct SparkPartitionIdRewriter<'s> {
     plan: LogicalPlan,
diff --git a/crates/sail-plan/src/resolver/tree/table_input.rs b/crates/sail-plan/src/resolver/tree/table_input.rs
index ce419edf3..94d4134ac 100644
--- a/crates/sail-plan/src/resolver/tree/table_input.rs
+++ b/crates/sail-plan/src/resolver/tree/table_input.rs
@@ -1,15 +1,15 @@
 use std::mem;
 
 use datafusion_common::tree_node::{Transformed, TreeNodeRewriter};
-use datafusion_common::{plan_datafusion_err, Result};
+use datafusion_common::{Result, plan_datafusion_err};
 use datafusion_expr::expr::ScalarFunction;
 use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, ScalarUDF};
 use sail_function::scalar::struct_function::StructFunction;
 use sail_function::scalar::table_input::TableInput;
 
-use crate::resolver::state::PlanResolverState;
-use crate::resolver::tree::{empty_logical_plan, PlanRewriter};
 use crate::resolver::PlanResolver;
+use crate::resolver::state::PlanResolverState;
+use crate::resolver::tree::{PlanRewriter, empty_logical_plan};
 
 /// Rewrites table input expression as cross join in UDTF lateral view.
 pub(crate) struct TableInputRewriter<'s> {
diff --git a/crates/sail-plan/src/resolver/tree/window.rs b/crates/sail-plan/src/resolver/tree/window.rs
index 93106fdfe..b028d179d 100644
--- a/crates/sail-plan/src/resolver/tree/window.rs
+++ b/crates/sail-plan/src/resolver/tree/window.rs
@@ -3,12 +3,22 @@ use std::mem;
 use std::sync::Arc;
 
 use datafusion::common::Result;
+use datafusion::functions_window::nth_value::{NthValue, NthValueKind};
 use datafusion::logical_expr::logical_plan::Window;
 use datafusion_common::tree_node::{Transformed, TreeNodeRewriter};
-use datafusion_expr::{ident, Expr, LogicalPlan};
+use datafusion_common::{DFSchemaRef, DataFusionError};
+use datafusion_expr::expr::{WindowFunction, WindowFunctionDefinition};
+use datafusion_expr::{
+    Expr, Extension, LogicalPlan, SortExpr, WindowFrame, WindowFrameUnits, ident,
+};
+use sail_function::window::SparkFirstLastValue;
+use sail_logical_plan::monotonic_id::MonotonicIdNode;
+use sail_logical_plan::sort::RequiredSortNode;
 
+use crate::error::PlanResult;
+use crate::resolver::PlanResolver;
 use crate::resolver::state::PlanResolverState;
-use crate::resolver::tree::{empty_logical_plan, PlanRewriter};
+use crate::resolver::tree::{PlanRewriter, empty_logical_plan};
 
 pub(crate) struct WindowRewriter<'s> {
     plan: LogicalPlan,
@@ -18,6 +28,33 @@ pub(crate) struct WindowRewriter<'s> {
     /// multiple times in the expression tree (e.g. when a division-by-zero
     /// guard wraps a window-function divisor in a CASE expression).
     seen: HashMap,
+    input_order_initialized: bool,
+    stable_ordering: Option>,
+    stable_ordering_enforced: bool,
+}
+
+impl WindowRewriter<'_> {
+    fn requires_input_order(function: &WindowFunction, schema: &DFSchemaRef) -> PlanResult {
+        let function_requires_order = match &function.fun {
+            WindowFunctionDefinition::AggregateUDF(udaf) => {
+                !PlanResolver::is_order_irrelevant_udaf(
+                    udaf.as_ref(),
+                    &function.params.args,
+                    schema,
+                )?
+            }
+            WindowFunctionDefinition::WindowUDF(udwf) => {
+                udwf.inner().downcast_ref::().is_some_and(|nth| {
+                    matches!(nth.kind(), NthValueKind::First | NthValueKind::Last)
+                }) || udwf.inner().is::()
+            }
+        };
+        let frame_requires_order =
+            matches!(&function.fun, WindowFunctionDefinition::AggregateUDF(_))
+                && function.params.window_frame.units == WindowFrameUnits::Rows
+                && function.params.window_frame != WindowFrame::new(None);
+        Ok(function_requires_order || frame_requires_order)
+    }
 }
 
 impl<'s> PlanRewriter<'s> for WindowRewriter<'s> {
@@ -26,6 +63,9 @@ impl<'s> PlanRewriter<'s> for WindowRewriter<'s> {
             plan,
             state,
             seen: HashMap::new(),
+            input_order_initialized: false,
+            stable_ordering: None,
+            stable_ordering_enforced: false,
         }
     }
 
@@ -39,7 +79,82 @@ impl TreeNodeRewriter for WindowRewriter<'_> {
 
     fn f_up(&mut self, node: Expr) -> Result> {
         match node {
-            Expr::WindowFunction(_) => {
+            Expr::WindowFunction(function) => {
+                let requires_input_order =
+                    Self::requires_input_order(&function, self.plan.schema())
+                        .map_err(|error| DataFusionError::External(Box::new(error)))?;
+
+                if !self.input_order_initialized {
+                    self.input_order_initialized = true;
+                    let (plan, found, global) =
+                        PlanResolver::require_input_sort_inner(self.plan.clone())
+                            .map_err(|error| DataFusionError::External(Box::new(error)))?;
+                    self.plan = plan;
+                    if found && global {
+                        let mut ordering =
+                            PlanResolver::input_sort_ordering(&self.plan).unwrap_or_default();
+                        let column = self.state.register_hidden_field_name("");
+                        self.plan = LogicalPlan::Extension(Extension {
+                            node: Arc::new(MonotonicIdNode::try_new(
+                                Arc::new(self.plan.clone()),
+                                column.clone(),
+                            )?),
+                        });
+                        ordering.push(SortExpr {
+                            expr: ident(column),
+                            asc: true,
+                            nulls_first: true,
+                        });
+                        self.stable_ordering = Some(ordering);
+                        self.stable_ordering_enforced = true;
+                    }
+                }
+
+                let mut required = function
+                    .params
+                    .partition_by
+                    .iter()
+                    .cloned()
+                    .map(|expr| SortExpr {
+                        expr,
+                        asc: true,
+                        nulls_first: true,
+                    })
+                    .collect::>();
+                required.extend(function.params.order_by.clone());
+
+                if let Some(prior) = self.stable_ordering.clone() {
+                    let prior_satisfies_required =
+                        required.is_empty() || prior.starts_with(&required);
+                    let already_enforced =
+                        self.stable_ordering_enforced && prior_satisfies_required;
+                    let desired = if prior_satisfies_required {
+                        prior
+                    } else {
+                        for sort in prior {
+                            if !required.iter().any(|x| x.expr == sort.expr) {
+                                required.push(sort);
+                            }
+                        }
+                        required
+                    };
+
+                    if requires_input_order && !already_enforced {
+                        self.plan = LogicalPlan::Extension(Extension {
+                            node: Arc::new(RequiredSortNode::new(
+                                Arc::new(self.plan.clone()),
+                                desired.clone(),
+                                None,
+                                false,
+                            )),
+                        });
+                        self.stable_ordering_enforced = true;
+                    } else if !already_enforced {
+                        self.stable_ordering_enforced = false;
+                    }
+                    self.stable_ordering = Some(desired);
+                }
+                let node = Expr::WindowFunction(function);
                 let name = node.schema_name().to_string();
                 if let Some(replacement) = self.seen.get(&name) {
                     return Ok(Transformed::yes(replacement.clone()));
diff --git a/crates/sail-plan/src/streaming/rewriter.rs b/crates/sail-plan/src/streaming/rewriter.rs
index 12c267c86..e55a3bcb7 100644
--- a/crates/sail-plan/src/streaming/rewriter.rs
+++ b/crates/sail-plan/src/streaming/rewriter.rs
@@ -1,17 +1,17 @@
 use std::sync::Arc;
 
 use datafusion::common::tree_node::TreeNode;
-use datafusion::datasource::{source_as_provider, TableProvider};
+use datafusion::datasource::{TableProvider, source_as_provider};
 use datafusion::logical_expr::{Extension, LogicalPlan};
 use datafusion_common::tree_node::{Transformed, TreeNodeRewriter};
-use datafusion_common::{internal_err, not_impl_err, plan_err, Result};
+use datafusion_common::{Result, internal_err, not_impl_err, plan_err};
 use datafusion_expr::{
-    col, or, Explain, FetchType, Filter, Projection, SkipType, SubqueryAlias, TableScan, Union,
-    UserDefinedLogicalNode,
+    Explain, FetchType, Filter, Projection, SkipType, SubqueryAlias, TableScan, Union,
+    UserDefinedLogicalNode, col, or,
 };
 use sail_common_datafusion::rename::table_provider::RenameTableProvider;
 use sail_common_datafusion::streaming::event::schema::{
-    is_flow_event_schema, MARKER_FIELD_NAME, RETRACTED_FIELD_NAME,
+    MARKER_FIELD_NAME, RETRACTED_FIELD_NAME, is_flow_event_schema,
 };
 use sail_common_datafusion::streaming::source::{StreamSource, StreamSourceTableProvider};
 use sail_data_source::formats::console::ConsoleWriteNode;
@@ -102,9 +102,9 @@ impl TreeNodeRewriter for StreamingRewriter {
             LogicalPlan::Repartition(_) => {
                 not_impl_err!("streaming repartition: {plan:?}")
             }
-            LogicalPlan::TableScan(ref scan) => {
-                if let Ok(provider) = source_as_provider(&scan.source) {
-                    if let Some(source) = get_stream_source_opt(provider.as_ref()) {
+            LogicalPlan::TableScan(ref scan) => match source_as_provider(&scan.source) {
+                Ok(provider) => match get_stream_source_opt(provider.as_ref()) {
+                    Some(source) => {
                         let NamedStreamSource { source, names } = source;
                         let TableScan {
                             table_name,
@@ -124,17 +124,15 @@ impl TreeNodeRewriter for StreamingRewriter {
                                 *fetch,
                             )?),
                         })))
-                    } else {
-                        Ok(Transformed::yes(LogicalPlan::Extension(Extension {
-                            node: Arc::new(StreamSourceAdapterNode::try_new(Arc::new(plan))?),
-                        })))
                     }
-                } else {
-                    Ok(Transformed::yes(LogicalPlan::Extension(Extension {
+                    _ => Ok(Transformed::yes(LogicalPlan::Extension(Extension {
                         node: Arc::new(StreamSourceAdapterNode::try_new(Arc::new(plan))?),
-                    })))
-                }
-            }
+                    }))),
+                },
+                _ => Ok(Transformed::yes(LogicalPlan::Extension(Extension {
+                    node: Arc::new(StreamSourceAdapterNode::try_new(Arc::new(plan))?),
+                }))),
+            },
             LogicalPlan::Union(union) => Ok(Transformed::yes(LogicalPlan::Union(
                 Union::try_new_with_loose_types(union.inputs)?,
             ))),
@@ -212,8 +210,8 @@ fn get_stream_source_opt(provider: &dyn TableProvider) -> Option() {
-        if let Some(stream) = get_stream_source_opt(rename.inner().as_ref()) {
-            Some(NamedStreamSource {
+        match get_stream_source_opt(rename.inner().as_ref()) {
+            Some(stream) => Some(NamedStreamSource {
                 source: stream.source,
                 names: Some(
                     rename
@@ -223,9 +221,8 @@ fn get_stream_source_opt(provider: &dyn TableProvider) -> Option None,
         }
     } else {
         None
diff --git a/crates/sail-pyarrow/src/lib.rs b/crates/sail-pyarrow/src/lib.rs
index d9a02bde0..6d27a8022 100644
--- a/crates/sail-pyarrow/src/lib.rs
+++ b/crates/sail-pyarrow/src/lib.rs
@@ -66,8 +66,8 @@ use std::sync::Arc;
 use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
 use arrow_array::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
 use arrow_array::{
-    ffi, make_array, RecordBatch, RecordBatchIterator, RecordBatchOptions, RecordBatchReader,
-    StructArray,
+    RecordBatch, RecordBatchIterator, RecordBatchOptions, RecordBatchReader, StructArray, ffi,
+    make_array,
 };
 use arrow_data::ArrayData;
 use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
diff --git a/crates/sail-python-udf/src/accumulator.rs b/crates/sail-python-udf/src/accumulator.rs
index 267b422f6..bd0073e2f 100644
--- a/crates/sail-python-udf/src/accumulator.rs
+++ b/crates/sail-python-udf/src/accumulator.rs
@@ -1,12 +1,12 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
-use datafusion::arrow::array::{new_empty_array, ArrayRef, AsArray, ListArray};
+use datafusion::arrow::array::{ArrayRef, AsArray, ListArray, new_empty_array};
 use datafusion::arrow::buffer::OffsetBuffer;
 use datafusion::arrow::compute::concat;
 use datafusion::arrow::datatypes::{DataType, Field, FieldRef};
 use datafusion::logical_expr::Accumulator;
-use datafusion_common::{exec_err, Result, ScalarValue};
+use datafusion_common::{Result, ScalarValue, exec_err};
 use datafusion_expr::function::StateFieldsArgs;
 use datafusion_expr::utils::format_state_name;
 
diff --git a/crates/sail-python-udf/src/array.rs b/crates/sail-python-udf/src/array.rs
index 200151480..8c4b88dba 100644
--- a/crates/sail-python-udf/src/array.rs
+++ b/crates/sail-python-udf/src/array.rs
@@ -4,7 +4,7 @@ use datafusion::arrow::array::{Array, ArrayRef, ListArray};
 use datafusion::arrow::buffer::OffsetBuffer;
 use datafusion::arrow::compute::concat;
 use datafusion::arrow::datatypes::{DataType, Field, FieldRef};
-use datafusion_common::{plan_err, Result};
+use datafusion_common::{Result, plan_err};
 
 pub fn get_fields(data_types: &[DataType], names: &[String]) -> Result> {
     if data_types.len() != names.len() {
diff --git a/crates/sail-python-udf/src/cereal/mod.rs b/crates/sail-python-udf/src/cereal/mod.rs
index 3c5aa14e4..ab72fd812 100644
--- a/crates/sail-python-udf/src/cereal/mod.rs
+++ b/crates/sail-python-udf/src/cereal/mod.rs
@@ -1,45 +1,44 @@
 use datafusion::arrow::datatypes::DataType;
 use pyo3::prelude::PyAnyMethods;
+use pyo3::sync::PyOnceLock;
 use pyo3::types::{PyList, PyModule};
-use pyo3::{intern, PyResult, Python};
+use pyo3::{PyResult, Python, intern};
 use sail_common::spec;
-use sail_pyarrow::ToPyArrow;
 
+use crate::conversion::TryToPy;
 use crate::error::{PyUdfError, PyUdfResult};
 
 pub mod pyspark_udf;
 pub mod pyspark_udtf;
 
+#[derive(Clone, Copy, PartialEq, Eq)]
 enum PySparkVersion {
     V3,
     V4_0,
     V4_1,
-}
-
-impl PySparkVersion {
-    fn is_v4(&self) -> bool {
-        matches!(self, PySparkVersion::V4_0 | PySparkVersion::V4_1)
-    }
+    V4_2,
 }
 
 fn get_pyspark_version() -> PyUdfResult {
-    use pyo3::prelude::PyAnyMethods;
-    use pyo3::types::PyModule;
+    static PYSPARK_VERSION: PyOnceLock = PyOnceLock::new();
 
     Python::attach(|py| {
-        let module = PyModule::import(py, "pyspark")?;
-        let version: String = module.getattr("__version__")?.extract()?;
-        if version.starts_with("3.") {
-            Ok(PySparkVersion::V3)
-        } else if version.starts_with("4.0.") {
-            Ok(PySparkVersion::V4_0)
-        } else if version.starts_with("4.") {
-            Ok(PySparkVersion::V4_1)
-        } else {
-            Err(PyUdfError::invalid(format!(
-                "unsupported PySpark version: {version}"
-            )))
-        }
+        PYSPARK_VERSION
+            .get_or_try_init(py, || {
+                let module = PyModule::import(py, "pyspark")?;
+                let version: String = module.getattr("__version__")?.extract()?;
+                let parts: Vec<_> = version.split('.').collect();
+                match parts.as_slice() {
+                    ["3", ..] => Ok(PySparkVersion::V3),
+                    ["4", "0", ..] => Ok(PySparkVersion::V4_0),
+                    ["4", "1", ..] => Ok(PySparkVersion::V4_1),
+                    ["4", "2", ..] => Ok(PySparkVersion::V4_2),
+                    _ => Err(PyUdfError::invalid(format!(
+                        "unsupported PySpark version: {version}"
+                    ))),
+                }
+            })
+            .copied()
     })
 }
 
@@ -104,6 +103,16 @@ pub(crate) fn write_kwarg(data: &mut Vec, kwargs: &[Option], index:
     }
 }
 
+pub(crate) fn write_conf(data: &mut Vec, conf: Vec<(String, String)>) {
+    data.extend((conf.len() as i32).to_be_bytes());
+    for (key, value) in conf {
+        data.extend((key.len() as i32).to_be_bytes());
+        data.extend(key.as_bytes());
+        data.extend((value.len() as i32).to_be_bytes());
+        data.extend(value.as_bytes());
+    }
+}
+
 fn should_write_config(eval_type: spec::PySparkUdfType) -> bool {
     use spec::PySparkUdfType;
 
@@ -141,7 +150,7 @@ fn should_write_config(eval_type: spec::PySparkUdfType) -> bool {
 /// Builds a JSON string representing a PySpark StructType schema from Arrow input types.
 /// This is used by PySpark 4.x's `read_udfs` to deserialize input type information
 /// for `SQL_ARROW_BATCHED_UDF`.
-fn build_input_types_json(input_types: &[DataType]) -> PyUdfResult {
+fn build_input_types_json(input_types: &[DataType], large_var_types: bool) -> PyUdfResult {
     Python::attach(|py| -> PyResult {
         let types_module = PyModule::import(py, intern!(py, "pyspark.sql.types"))?;
         let struct_type_cls = types_module.getattr(intern!(py, "StructType"))?;
@@ -153,7 +162,7 @@ fn build_input_types_json(input_types: &[DataType]) -> PyUdfResult {
             .iter()
             .enumerate()
             .map(|(i, dt)| -> PyResult<_> {
-                let arrow_type = dt.to_pyarrow(py)?;
+                let arrow_type = dt.try_to_py(py, large_var_types)?;
                 let spark_type = from_arrow_type.call1((arrow_type,))?;
                 struct_field_cls.call1((format!("_{i}"), spark_type, true))
             })
diff --git a/crates/sail-python-udf/src/cereal/pyspark_udf.rs b/crates/sail-python-udf/src/cereal/pyspark_udf.rs
index c6d32f490..dbf8850cc 100644
--- a/crates/sail-python-udf/src/cereal/pyspark_udf.rs
+++ b/crates/sail-python-udf/src/cereal/pyspark_udf.rs
@@ -2,12 +2,12 @@ use datafusion::arrow::datatypes::DataType;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::PyAnyMethods;
 use pyo3::types::PyModule;
-use pyo3::{intern, Bound, IntoPyObject, PyAny, Python};
+use pyo3::{Bound, IntoPyObject, PyAny, Python, intern};
 use sail_common::spec;
 
 use crate::cereal::{
-    build_input_types_json, check_python_udf_version, get_pyspark_version, should_write_config,
-    supports_kwargs, write_kwarg, PySparkVersion,
+    PySparkVersion, build_input_types_json, check_python_udf_version, get_pyspark_version,
+    should_write_config, supports_kwargs, write_conf, write_kwarg,
 };
 use crate::config::PySparkUdfConfig;
 use crate::error::{PyUdfError, PyUdfResult};
@@ -29,9 +29,20 @@ impl PySparkUdfPayload {
         let serializer = PyModule::import(py, intern!(py, "pyspark.serializers"))?
             .getattr(intern!(py, "CPickleSerializer"))?
             .call0()?;
-        let tuple = PyModule::import(py, intern!(py, "pyspark.worker"))?
-            .getattr(intern!(py, "read_udfs"))?
-            .call1((serializer, infile, eval_type))?;
+        let worker = PyModule::import(py, intern!(py, "pyspark.worker"))?;
+        let read_udfs = worker.getattr(intern!(py, "read_udfs"))?;
+        let tuple = match get_pyspark_version()? {
+            PySparkVersion::V4_2 => {
+                let runner_conf = worker
+                    .getattr(intern!(py, "RunnerConf"))?
+                    .call1((&infile,))?;
+                let eval_conf = worker.getattr(intern!(py, "EvalConf"))?.call1((&infile,))?;
+                read_udfs.call1((serializer, infile, eval_type, runner_conf, eval_conf))?
+            }
+            PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => {
+                read_udfs.call1((serializer, infile, eval_type))?
+            }
+        };
         tuple
             .get_item(0)?
             .into_pyobject(py)
@@ -54,29 +65,42 @@ impl PySparkUdfPayload {
 
         data.extend(i32::from(eval_type).to_be_bytes());
 
-        if should_write_config(eval_type) {
-            let config = config.to_key_value_pairs();
-            data.extend((config.len() as i32).to_be_bytes()); // number of configuration options
-            for (key, value) in config {
-                data.extend((key.len() as i32).to_be_bytes()); // length of the key
-                data.extend(key.as_bytes());
-                data.extend((value.len() as i32).to_be_bytes()); // length of the value
-                data.extend(value.as_bytes());
+        match pyspark_version {
+            PySparkVersion::V4_2 => {
+                // Spark 4.2 reads both maps before dispatching to read_udfs.
+                write_conf(&mut data, config.to_key_value_pairs());
+                let mut eval_conf = vec![];
+                if eval_type == spec::PySparkUdfType::ArrowBatched {
+                    eval_conf.push((
+                        "input_type".to_string(),
+                        build_input_types_json(input_types, config.arrow_use_large_var_types)?,
+                    ))
+                }
+                write_conf(&mut data, eval_conf);
             }
-        }
+            PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => {
+                if should_write_config(eval_type) {
+                    write_conf(&mut data, config.to_key_value_pairs());
+                }
 
-        // PySpark 4.1+ reads input types for ArrowBatched UDFs.
-        // PySpark 4.0.x does not read input types and would misparse the stream.
-        if matches!(pyspark_version, PySparkVersion::V4_1)
-            && matches!(eval_type, spec::PySparkUdfType::ArrowBatched)
-        {
-            let schema_json = build_input_types_json(input_types)?;
-            data.extend((schema_json.len() as i32).to_be_bytes());
-            data.extend(schema_json.as_bytes());
+                // PySpark 4.1 reads input types for ArrowBatched UDFs.
+                // PySpark 4.0.x does not read input types and would misparse the stream.
+                if pyspark_version == PySparkVersion::V4_1
+                    && eval_type == spec::PySparkUdfType::ArrowBatched
+                {
+                    let schema_json =
+                        build_input_types_json(input_types, config.arrow_use_large_var_types)?;
+                    data.extend((schema_json.len() as i32).to_be_bytes());
+                    data.extend(schema_json.as_bytes());
+                }
+            }
         }
 
-        if pyspark_version.is_v4() {
-            data.extend(0u8.to_be_bytes()); // profiling is not enabled
+        match pyspark_version {
+            PySparkVersion::V4_0 | PySparkVersion::V4_1 => {
+                data.extend(0u8.to_be_bytes()); // profiling is not enabled
+            }
+            PySparkVersion::V3 | PySparkVersion::V4_2 => {}
         }
 
         data.extend(1i32.to_be_bytes()); // number of UDFs
@@ -87,7 +111,11 @@ impl PySparkUdfPayload {
             .map_err(|e| PyUdfError::invalid(format!("num args: {e}")))?;
         data.extend(num_arg_offsets.to_be_bytes()); // number of argument offsets
 
-        let allow_kwargs = pyspark_version.is_v4() && supports_kwargs(eval_type);
+        let allow_kwargs = match pyspark_version {
+            PySparkVersion::V4_2 => true,
+            PySparkVersion::V4_0 | PySparkVersion::V4_1 => supports_kwargs(eval_type),
+            PySparkVersion::V3 => false,
+        };
 
         for (i, offset) in arg_offsets.iter().enumerate() {
             let offset: i32 = (*offset)
@@ -103,6 +131,14 @@ impl PySparkUdfPayload {
         data.extend((command.len() as i32).to_be_bytes()); // length of the function
         data.extend_from_slice(command);
 
+        match pyspark_version {
+            PySparkVersion::V4_2 => {
+                // Spark 4.2 always sends this field, even with profiling disabled.
+                data.extend(0i64.to_be_bytes());
+            }
+            PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => {}
+        }
+
         Ok(data)
     }
 }
diff --git a/crates/sail-python-udf/src/cereal/pyspark_udtf.rs b/crates/sail-python-udf/src/cereal/pyspark_udtf.rs
index 7325e9eb4..5d812de98 100644
--- a/crates/sail-python-udf/src/cereal/pyspark_udtf.rs
+++ b/crates/sail-python-udf/src/cereal/pyspark_udtf.rs
@@ -3,15 +3,16 @@ use datafusion_common::ScalarValue;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::PyAnyMethods;
 use pyo3::types::PyModule;
-use pyo3::{intern, Bound, IntoPyObject, PyAny, PyResult, Python};
+use pyo3::{Bound, IntoPyObject, PyAny, PyResult, Python, intern};
 use sail_common::spec;
-use sail_pyarrow::{FromPyArrow, ToPyArrow};
+use sail_pyarrow::FromPyArrow;
 
 use crate::cereal::{
-    build_input_types_json, check_python_udf_version, get_pyspark_version, should_write_config,
-    supports_kwargs, write_kwarg, PySparkVersion,
+    PySparkVersion, build_input_types_json, check_python_udf_version, get_pyspark_version,
+    should_write_config, supports_kwargs, write_conf, write_kwarg,
 };
 use crate::config::PySparkUdfConfig;
+use crate::conversion::TryToPy;
 use crate::error::{PyUdfError, PyUdfResult};
 
 pub struct PySparkUdtfPayload;
@@ -31,9 +32,20 @@ impl PySparkUdtfPayload {
         let serializer = PyModule::import(py, intern!(py, "pyspark.serializers"))?
             .getattr(intern!(py, "CPickleSerializer"))?
             .call0()?;
-        let tuple = PyModule::import(py, intern!(py, "pyspark.worker"))?
-            .getattr(intern!(py, "read_udtf"))?
-            .call1((serializer, infile, eval_type))?;
+        let worker = PyModule::import(py, intern!(py, "pyspark.worker"))?;
+        let read_udtf = worker.getattr(intern!(py, "read_udtf"))?;
+        let tuple = match get_pyspark_version()? {
+            PySparkVersion::V4_2 => {
+                let runner_conf = worker
+                    .getattr(intern!(py, "RunnerConf"))?
+                    .call1((&infile,))?;
+                let eval_conf = worker.getattr(intern!(py, "EvalConf"))?.call1((&infile,))?;
+                read_udtf.call1((serializer, infile, eval_type, runner_conf, eval_conf))?
+            }
+            PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => {
+                read_udtf.call1((serializer, infile, eval_type))?
+            }
+        };
         tuple
             .get_item(0)?
             .into_pyobject(py)
@@ -63,6 +75,7 @@ impl PySparkUdtfPayload {
         argument_literals: &[Option],
         kwargs: &[Option],
         argument_is_tables: &[bool],
+        large_var_types: bool,
     ) -> PyUdfResult {
         check_python_udf_version(python_version)?;
         let _ = eval_type; // eval_type is not needed for the analyze call itself
@@ -81,7 +94,9 @@ impl PySparkUdtfPayload {
             // Build the list of arguments: (arrow_type, is_constant, value_array, kwarg_name, is_table)
             let mut arguments: Vec> = Vec::with_capacity(argument_types.len());
             for (i, dt) in argument_types.iter().enumerate() {
-                let arrow_type = dt.to_pyarrow(py).map_err(PyUdfError::PythonError)?;
+                let arrow_type = dt
+                    .try_to_py(py, large_var_types)
+                    .map_err(PyUdfError::PythonError)?;
                 let (is_constant, value_array) = match argument_literals.get(i) {
                     Some(Some(sv)) => {
                         // Constant expression: create a single-element PyArrow array.
@@ -92,8 +107,7 @@ impl PySparkUdtfPayload {
                             .map_err(|e| PyValueError::new_err(e.to_string()))
                             .map_err(PyUdfError::PythonError)?;
                         let pyarrow_array = array
-                            .to_data()
-                            .to_pyarrow(py)
+                            .try_to_py(py, large_var_types)
                             .map_err(PyUdfError::PythonError)?;
                         (true, Some(pyarrow_array))
                     }
@@ -126,12 +140,14 @@ impl PySparkUdtfPayload {
         })
     }
 
+    #[expect(clippy::too_many_arguments)]
     pub fn build(
         python_version: &str,
         command: &[u8],
         eval_type: spec::PySparkUdfType,
         num_args: usize,
         input_types: &[DataType],
+        passthrough_columns: usize,
         kwargs: &[Option],
         return_type: &DataType,
         config: &PySparkUdfConfig,
@@ -142,39 +158,68 @@ impl PySparkUdtfPayload {
 
         data.extend(i32::from(eval_type).to_be_bytes()); // Add eval_type for extraction in visit_bytes
 
-        if should_write_config(eval_type) {
-            let config = config.to_key_value_pairs();
-            data.extend((config.len() as i32).to_be_bytes()); // number of configuration options
-            for (key, value) in config {
-                data.extend((key.len() as i32).to_be_bytes()); // length of the key
-                data.extend(key.as_bytes());
-                data.extend((value.len() as i32).to_be_bytes()); // length of the value
-                data.extend(value.as_bytes());
+        match pyspark_version {
+            PySparkVersion::V4_2 => {
+                write_conf(&mut data, config.to_key_value_pairs());
+                let mut eval_conf = vec![];
+                if eval_type == spec::PySparkUdfType::ArrowTable {
+                    let argument_types = input_types
+                        .get(passthrough_columns..)
+                        .ok_or_else(|| {
+                            PyUdfError::invalid(format!(
+                                "passthrough columns ({passthrough_columns}) exceed input columns ({})",
+                                input_types.len()
+                            ))
+                        })?;
+                    eval_conf.push((
+                        "input_type".to_string(),
+                        build_input_types_json(argument_types, config.arrow_use_large_var_types)?,
+                    ))
+                }
+                write_conf(&mut data, eval_conf);
             }
-        }
+            PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => {
+                if should_write_config(eval_type) {
+                    write_conf(&mut data, config.to_key_value_pairs());
+                }
 
-        // PySpark 4.1+ reads input types for ArrowTable UDTFs.
-        // PySpark 4.0.x does not read input types and would misparse the stream.
-        if matches!(pyspark_version, PySparkVersion::V4_1)
-            && matches!(eval_type, spec::PySparkUdfType::ArrowTable)
-        {
-            let schema_json = build_input_types_json(input_types)?;
-            data.extend((schema_json.len() as i32).to_be_bytes());
-            data.extend(schema_json.as_bytes());
-        }
+                // PySpark 4.1+ reads input types for ArrowTable UDTFs.
+                // PySpark 4.0.x does not read input types and would misparse the stream.
+                if pyspark_version == PySparkVersion::V4_1
+                    && eval_type == spec::PySparkUdfType::ArrowTable
+                {
+                    let argument_types = input_types
+                        .get(passthrough_columns..)
+                        .ok_or_else(|| {
+                            PyUdfError::invalid(format!(
+                                "passthrough columns ({passthrough_columns}) exceed input columns ({})",
+                                input_types.len()
+                            ))
+                        })?;
+                    let schema_json =
+                        build_input_types_json(argument_types, config.arrow_use_large_var_types)?;
+                    data.extend((schema_json.len() as i32).to_be_bytes());
+                    data.extend(schema_json.as_bytes());
+                }
 
-        // PySpark 4.1+ reads table argument offsets for ArrowUdtf before the arg offsets.
-        // PySpark 4.0.x does not use the ArrowUdtf eval type.
-        if matches!(pyspark_version, PySparkVersion::V4_1)
-            && matches!(eval_type, spec::PySparkUdfType::ArrowUdtf)
-        {
-            data.extend(0i32.to_be_bytes()); // num_table_arg_offsets = 0
+                // PySpark 4.1+ reads table argument offsets for ArrowUdtf before the arg offsets.
+                // PySpark 4.0.x does not use the ArrowUdtf eval type.
+                if pyspark_version == PySparkVersion::V4_1
+                    && eval_type == spec::PySparkUdfType::ArrowUdtf
+                {
+                    data.extend(0i32.to_be_bytes()); // num_table_arg_offsets = 0
+                }
+            }
         }
 
         let num_args: i32 = num_args
             .try_into()
             .map_err(|e| PyUdfError::invalid(format!("num_args: {e}")))?;
-        let allow_kwargs = pyspark_version.is_v4() && supports_kwargs(eval_type);
+        let allow_kwargs = match pyspark_version {
+            PySparkVersion::V4_2 => true,
+            PySparkVersion::V4_0 | PySparkVersion::V4_1 => supports_kwargs(eval_type),
+            PySparkVersion::V3 => false,
+        };
         data.extend(num_args.to_be_bytes()); // number of arguments
         for index in 0..num_args {
             data.extend(index.to_be_bytes()); // argument offset
@@ -183,16 +228,19 @@ impl PySparkUdtfPayload {
             }
         }
 
-        if pyspark_version.is_v4() {
-            data.extend(0i32.to_be_bytes()); // number of partition child indexes
-            data.extend(0u8.to_be_bytes()); // pickled analyze result is not present
+        match pyspark_version {
+            PySparkVersion::V4_0 | PySparkVersion::V4_1 | PySparkVersion::V4_2 => {
+                data.extend(0i32.to_be_bytes()); // number of partition child indexes
+                data.extend(0u8.to_be_bytes()); // pickled analyze result is not present
+            }
+            PySparkVersion::V3 => {}
         }
 
         data.extend((command.len() as i32).to_be_bytes()); // length of the function
         data.extend_from_slice(command);
 
         let type_string = Python::attach(|py| -> PyResult {
-            let return_type = return_type.to_pyarrow(py)?;
+            let return_type = return_type.try_to_py(py, config.arrow_use_large_var_types)?;
             PyModule::import(py, intern!(py, "pyspark.sql.pandas.types"))?
                 .getattr(intern!(py, "from_arrow_type"))?
                 .call1((return_type,))?
@@ -203,9 +251,12 @@ impl PySparkUdtfPayload {
         data.extend((type_string.len() as u32).to_be_bytes());
         data.extend(type_string.as_bytes());
 
-        if pyspark_version.is_v4() {
-            // TODO: support UDTF name
-            data.extend(0u32.to_be_bytes()); // length of UDTF name
+        match pyspark_version {
+            PySparkVersion::V4_0 | PySparkVersion::V4_1 | PySparkVersion::V4_2 => {
+                // TODO: support UDTF name
+                data.extend(0u32.to_be_bytes()); // length of UDTF name
+            }
+            PySparkVersion::V3 => {}
         }
 
         Ok(data)
diff --git a/crates/sail-python-udf/src/config.rs b/crates/sail-python-udf/src/config.rs
index fa2b557ae..8ee8a5bc0 100644
--- a/crates/sail-python-udf/src/config.rs
+++ b/crates/sail-python-udf/src/config.rs
@@ -14,12 +14,16 @@ pub struct PySparkUdfConfig {
     #[pyo3(get)]
     pub arrow_max_records_per_batch: usize,
     #[pyo3(get)]
+    pub arrow_use_large_var_types: bool,
+    #[pyo3(get)]
     pub python_udf_pandas_conversion_enabled: bool,
     #[pyo3(get)]
     pub python_udtf_pandas_conversion_enabled: bool,
     #[pyo3(get)]
     pub python_udf_pandas_int_to_decimal_coercion_enabled: bool,
     #[pyo3(get)]
+    pub python_udf_pandas_prefer_int_extension_dtype: bool,
+    #[pyo3(get)]
     pub binary_as_bytes: bool,
 }
 
@@ -31,9 +35,11 @@ impl Default for PySparkUdfConfig {
             pandas_grouped_map_assign_columns_by_name: true,
             pandas_convert_to_arrow_array_safely: false,
             arrow_max_records_per_batch: 10000,
+            arrow_use_large_var_types: false,
             python_udf_pandas_conversion_enabled: false,
             python_udtf_pandas_conversion_enabled: false,
             python_udf_pandas_int_to_decimal_coercion_enabled: false,
+            python_udf_pandas_prefer_int_extension_dtype: false,
             binary_as_bytes: true,
         }
     }
@@ -69,6 +75,10 @@ impl PySparkUdfConfig {
             "spark.sql.execution.arrow.maxRecordsPerBatch".to_string(),
             self.arrow_max_records_per_batch.to_string(),
         ));
+        out.push((
+            "spark.sql.execution.arrow.useLargeVarTypes".to_string(),
+            self.arrow_use_large_var_types.to_string(),
+        ));
         out.push((
             "spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled".to_string(),
             self.python_udf_pandas_conversion_enabled.to_string(),
@@ -82,6 +92,11 @@ impl PySparkUdfConfig {
             self.python_udf_pandas_int_to_decimal_coercion_enabled
                 .to_string(),
         ));
+        out.push((
+            "spark.sql.execution.pythonUDF.pandas.preferIntExtensionDtype".to_string(),
+            self.python_udf_pandas_prefer_int_extension_dtype
+                .to_string(),
+        ));
         out.push((
             "spark.sql.execution.pyspark.binaryAsBytes".to_string(),
             self.binary_as_bytes.to_string(),
diff --git a/crates/sail-python-udf/src/conversion.rs b/crates/sail-python-udf/src/conversion.rs
index e60fbcaca..109ce8e53 100644
--- a/crates/sail-python-udf/src/conversion.rs
+++ b/crates/sail-python-udf/src/conversion.rs
@@ -1,16 +1,107 @@
+use std::sync::Arc;
+
 use datafusion::arrow::array::{Array, ArrayRef, RecordBatch};
-use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef};
+use datafusion::arrow::datatypes::{DataType, FieldRef, Schema, SchemaRef};
 use datafusion_common::arrow::array::ArrayData;
+use pyo3::exceptions::PyRuntimeError;
 use pyo3::{Bound, BoundObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python};
+use sail_common_datafusion::array::record_batch::{
+    cast_array_recursively, cast_record_batch_positionally,
+};
 use sail_pyarrow::{FromPyArrow, ToPyArrow};
 
+fn normalize_field_with_options(field: &FieldRef, large_var_types: bool) -> FieldRef {
+    Arc::new(
+        field
+            .as_ref()
+            .clone()
+            .with_data_type(normalize_data_type(field.data_type(), large_var_types)),
+    )
+}
+
+fn normalize_data_type(data_type: &DataType, large_var_types: bool) -> DataType {
+    match data_type {
+        DataType::Binary | DataType::LargeBinary | DataType::BinaryView if large_var_types => {
+            DataType::LargeBinary
+        }
+        DataType::Binary | DataType::LargeBinary | DataType::BinaryView => DataType::Binary,
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View if large_var_types => {
+            DataType::LargeUtf8
+        }
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
+        DataType::List(field) => {
+            DataType::List(normalize_field_with_options(field, large_var_types))
+        }
+        DataType::ListView(field) => {
+            DataType::List(normalize_field_with_options(field, large_var_types))
+        }
+        DataType::FixedSizeList(field, size) => {
+            DataType::FixedSizeList(normalize_field_with_options(field, large_var_types), *size)
+        }
+        DataType::LargeList(field) => {
+            DataType::LargeList(normalize_field_with_options(field, large_var_types))
+        }
+        DataType::LargeListView(field) => {
+            DataType::LargeList(normalize_field_with_options(field, large_var_types))
+        }
+        DataType::Struct(fields) => DataType::Struct(
+            fields
+                .iter()
+                .map(|field| normalize_field_with_options(field, large_var_types))
+                .collect(),
+        ),
+        DataType::Dictionary(_, value) => normalize_data_type(value, large_var_types),
+        DataType::Map(field, sorted) => DataType::Map(
+            normalize_field_with_options(field, large_var_types),
+            *sorted,
+        ),
+        DataType::RunEndEncoded(_, values) => {
+            normalize_data_type(values.data_type(), large_var_types)
+        }
+        _ => data_type.clone(),
+    }
+}
+
+fn normalize_schema(schema: &Schema, large_var_types: bool) -> Schema {
+    Schema::new_with_metadata(
+        schema
+            .fields()
+            .iter()
+            .map(|field| normalize_field_with_options(field, large_var_types))
+            .collect::>(),
+        schema.metadata().clone(),
+    )
+}
+
+fn normalize_array(array: &ArrayRef, large_var_types: bool) -> PyResult {
+    cast_array_recursively(
+        array,
+        &normalize_data_type(array.data_type(), large_var_types),
+    )
+    .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+}
+
+fn normalize_record_batch(batch: &RecordBatch, large_var_types: bool) -> PyResult {
+    let schema = Arc::new(normalize_schema(batch.schema().as_ref(), large_var_types));
+    if schema.as_ref() == batch.schema().as_ref() {
+        Ok(batch.clone())
+    } else {
+        cast_record_batch_positionally(batch.clone(), schema)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+}
+
 /// A trait that defines the custom behavior of converting Rust data to a Python object.
 pub trait TryToPy<'py> {
     type Target;
     type Output: BoundObject<'py, Self::Target>;
     type Error: Into;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result;
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result;
 }
 
 impl<'py> TryToPy<'py> for &DataType {
@@ -18,8 +109,14 @@ impl<'py> TryToPy<'py> for &DataType {
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
-        self.to_pyarrow(py).map(|obj| obj.into_bound())
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
+        normalize_data_type(self, large_var_types)
+            .to_pyarrow(py)
+            .map(|obj| obj.into_bound())
     }
 }
 
@@ -28,22 +125,47 @@ impl<'py> TryToPy<'py> for &[DataType] {
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
         self.iter()
-            .map(|x| x.to_pyarrow(py))
+            .map(|x| normalize_data_type(x, large_var_types).to_pyarrow(py))
             .collect::>>()
             .map(|x| x.into_pyobject(py))?
     }
 }
 
+impl<'py> TryToPy<'py> for ArrayRef {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
+        normalize_array(self, large_var_types)?
+            .into_data()
+            .to_pyarrow(py)
+            .map(|obj| obj.into_bound())
+    }
+}
+
 impl<'py> TryToPy<'py> for &[ArrayRef] {
     type Target = PyAny;
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
         self.iter()
-            .map(|x| x.into_data().to_pyarrow(py))
+            .map(|x| x.try_to_py(py, large_var_types))
             .collect::>>()
             .map(|x| x.into_pyobject(py))?
     }
@@ -54,8 +176,12 @@ impl<'py> TryToPy<'py> for Vec {
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
-        self.as_slice().try_to_py(py)
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
+        self.as_slice().try_to_py(py, large_var_types)
     }
 }
 
@@ -64,8 +190,14 @@ impl<'py> TryToPy<'py> for &Schema {
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
-        self.to_pyarrow(py).map(|obj| obj.into_bound())
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
+        normalize_schema(self, large_var_types)
+            .to_pyarrow(py)
+            .map(|obj| obj.into_bound())
     }
 }
 
@@ -74,8 +206,14 @@ impl<'py> TryToPy<'py> for SchemaRef {
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
-        self.to_pyarrow(py).map(|obj| obj.into_bound())
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
+        normalize_schema(self.as_ref(), large_var_types)
+            .to_pyarrow(py)
+            .map(|obj| obj.into_bound())
     }
 }
 
@@ -84,8 +222,14 @@ impl<'py> TryToPy<'py> for RecordBatch {
     type Output = Bound<'py, Self::Target>;
     type Error = PyErr;
 
-    fn try_to_py(&self, py: Python<'py>) -> Result {
-        self.to_pyarrow(py).map(|obj| obj.into_bound())
+    fn try_to_py(
+        &self,
+        py: Python<'py>,
+        large_var_types: bool,
+    ) -> Result {
+        normalize_record_batch(self, large_var_types)?
+            .to_pyarrow(py)
+            .map(|obj| obj.into_bound())
     }
 }
 
diff --git a/crates/sail-python-udf/src/error.rs b/crates/sail-python-udf/src/error.rs
index cd7a051e5..8bbf3a049 100644
--- a/crates/sail-python-udf/src/error.rs
+++ b/crates/sail-python-udf/src/error.rs
@@ -1,6 +1,6 @@
 use datafusion_common::DataFusionError;
 use pyo3::prelude::{PyAnyMethods, PyModule};
-use pyo3::{intern, PyErr, PyResult, Python};
+use pyo3::{PyErr, PyResult, Python, intern};
 use sail_common_datafusion::error::{PythonErrorCause, PythonErrorCauseExtractor};
 use thiserror::Error;
 
diff --git a/crates/sail-python-udf/src/python/mod.rs b/crates/sail-python-udf/src/python/mod.rs
index 8c921ba83..72b72b870 100644
--- a/crates/sail-python-udf/src/python/mod.rs
+++ b/crates/sail-python-udf/src/python/mod.rs
@@ -1,6 +1,6 @@
 use pyo3::prelude::{PyAnyMethods, PyModule};
 use pyo3::types::{PyString, PyTuple};
-use pyo3::{intern, Bound, BoundObject, IntoPyObject, PyAny, PyResult};
+use pyo3::{Bound, BoundObject, IntoPyObject, PyAny, PyResult, intern};
 
 pub(crate) mod spark;
 
diff --git a/crates/sail-python-udf/src/python/spark.py b/crates/sail-python-udf/src/python/spark.py
index 2efce2f36..f11c31f04 100644
--- a/crates/sail-python-udf/src/python/spark.py
+++ b/crates/sail-python-udf/src/python/spark.py
@@ -13,7 +13,7 @@
 import pyspark
 from pyspark.sql.pandas.serializers import ArrowStreamPandasUDFSerializer, ArrowStreamPandasUDTFSerializer
 from pyspark.sql.pandas.types import from_arrow_type
-from pyspark.sql.types import Row
+from pyspark.sql.types import DataType, Row, StructField, StructType
 
 _PYARROW_HAS_VIEW_TYPES = all(hasattr(pa, x) for x in ("list_view", "large_list_view", "string_view", "binary_view"))
 
@@ -67,6 +67,15 @@ def batched(iterable, n):
     _LocalDataToArrowConversion = None
 
 
+PYSPARK_VERSION: tuple[int, ...] = tuple(int(part) for part in pyspark.__version__.split("."))
+
+
+def _pandas_serializer_kwargs(config) -> dict[str, bool]:
+    if PYSPARK_VERSION >= (4, 2):
+        return {"prefer_int_ext_dtype": config.python_udf_pandas_prefer_int_extension_dtype}
+    return {}
+
+
 class Converter:
     """
     A converter that converts between PySpark data and Arrow data.
@@ -477,24 +486,94 @@ def _field_values(self, data: Any) -> Sequence[Any]:
         return self._spark_data_type.toInternal(data)
 
 
-if pyspark.__version__.startswith(("3.", "4.0.")):
+if (3,) <= PYSPARK_VERSION < (4, 1):
 
     def _arrow_column_to_pandas(column: pa.Array, serializer: ArrowStreamPandasUDFSerializer):
         return serializer.arrow_to_pandas(column)
-else:
+elif (4, 1) <= PYSPARK_VERSION < (4, 2):
 
     def _arrow_column_to_pandas(column: pa.Array, serializer: ArrowStreamPandasUDFSerializer):
         return serializer.arrow_to_pandas(column, 0)
+else:
+
+    def _arrow_column_to_pandas(column: pa.Array, serializer: ArrowStreamPandasUDFSerializer):
+        from pyspark.sql.conversion import ArrowBatchTransformer
+
+        schema = StructType([StructField("_0", from_arrow_type(column.type), True)])
+        batch = pa.RecordBatch.from_arrays([column], ["_0"])
+        return ArrowBatchTransformer.to_pandas(
+            batch,
+            timezone=serializer._timezone,  # noqa: SLF001
+            schema=schema,
+            struct_in_pandas=serializer._struct_in_pandas,  # noqa: SLF001
+            ndarray_as_list=serializer._ndarray_as_list,  # noqa: SLF001
+            prefer_int_ext_dtype=serializer._prefer_int_ext_dtype,  # noqa: SLF001
+            df_for_struct=serializer._df_for_struct,  # noqa: SLF001
+        )[0]
+
+
+if (3,) <= PYSPARK_VERSION < (4, 2):
+
+    def _pandas_to_arrow_array(data, data_type: pa.DataType, serializer: ArrowStreamPandasUDFSerializer) -> pa.Array:
+        if (
+            serializer._struct_in_pandas == "dict"  # noqa: SLF001
+            and pa.types.is_struct(data_type)
+            and not _is_variant_struct_type(data_type)
+        ):
+            return serializer._create_struct_array(data, data_type)  # noqa: SLF001
+        return serializer._create_array(data, data_type, arrow_cast=serializer._arrow_cast)  # noqa: SLF001
+else:
+
+    def _pandas_to_arrow_array(data, data_type: pa.DataType, serializer: ArrowStreamPandasUDFSerializer) -> pa.Array:
+        from pyspark.sql.conversion import PandasToArrowConversion
+
+        # Spark 4.2's worker wrappers return the Spark return type, whereas
+        # prior versions return its PyArrow counterpart.
+        spark_type = data_type if isinstance(data_type, DataType) else from_arrow_type(data_type)
+        schema = StructType([StructField("_0", spark_type, True)])
+        return PandasToArrowConversion.convert(
+            [data],
+            schema,
+            timezone=serializer._timezone,  # noqa: SLF001
+            safecheck=serializer._safecheck,  # noqa: SLF001
+            arrow_cast=serializer._arrow_cast,  # noqa: SLF001
+            prefers_large_types=serializer._prefers_large_types,  # noqa: SLF001
+            assign_cols_by_name=serializer._assign_cols_by_name,  # noqa: SLF001
+            int_to_decimal_coercion_enabled=serializer._int_to_decimal_coercion_enabled,  # noqa: SLF001
+            ignore_unexpected_complex_type_values=serializer._ignore_unexpected_complex_type_values,  # noqa: SLF001
+            is_legacy=serializer._is_legacy,  # noqa: SLF001
+        ).column(0)
+
+
+def _record_batch(
+    args: Sequence[pa.Array], *, names: Sequence[str] | None = None, num_rows: int | None = None
+) -> pa.RecordBatch:
+    if args:
+        return pa.RecordBatch.from_arrays(args, names if names is not None else [f"_{i}" for i in range(len(args))])
+    # Older supported PyArrow releases do not expose a row-count argument on
+    # RecordBatch.from_arrays. Select away a temporary column instead.
+    return pa.RecordBatch.from_arrays([pa.nulls(num_rows or 0)], ["_"]).select([])
+
+
+def _output_column(output: Iterator[pa.RecordBatch]) -> pa.Array:
+    arrays = []
+    for batch in output:
+        if isinstance(batch, pa.Table):
+            arrays.extend(part.column(0) for part in batch.to_batches())
+        else:
+            arrays.append(batch.column(0))
+    if not arrays:
+        return pa.array([])
+    return arrays[0] if len(arrays) == 1 else pa.concat_arrays(arrays)
 
 
-def _pandas_to_arrow_array(data, data_type: pa.DataType, serializer: ArrowStreamPandasUDFSerializer) -> pa.Array:
-    if (
-        serializer._struct_in_pandas == "dict"  # noqa: SLF001
-        and pa.types.is_struct(data_type)
-        and not _is_variant_struct_type(data_type)
-    ):
-        return serializer._create_struct_array(data, data_type)  # noqa: SLF001
-    return serializer._create_array(data, data_type, arrow_cast=serializer._arrow_cast)  # noqa: SLF001
+def _wrap_struct(batch: pa.RecordBatch) -> pa.RecordBatch:
+    return pa.RecordBatch.from_arrays([batch.to_struct_array()], ["_0"])
+
+
+def _flatten_struct(batch: pa.RecordBatch) -> pa.RecordBatch:
+    struct = batch.column(0)
+    return pa.RecordBatch.from_arrays(struct.flatten(), schema=pa.schema(struct.type))
 
 
 def _arrow_columns_to_python(args: list[pa.Array], *, binary_as_bytes: bool = True) -> tuple:
@@ -603,7 +682,7 @@ def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array:
 class PySparkArrowBatchUdf:
     def __init__(self, udf: Callable[..., Any], config):
         self._udf = udf
-        self._use_legacy = config.python_udf_pandas_conversion_enabled or pyspark.__version__.startswith(("3.", "4.0."))
+        self._use_legacy = config.python_udf_pandas_conversion_enabled or (3,) <= PYSPARK_VERSION < (4, 1)
         if self._use_legacy:
             self._serializer = ArrowStreamPandasUDFSerializer(
                 timezone=config.session_timezone,
@@ -613,6 +692,7 @@ def __init__(self, udf: Callable[..., Any], config):
                 struct_in_pandas="row",
                 ndarray_as_list=True,
                 arrow_cast=True,
+                **_pandas_serializer_kwargs(config),
             )
         else:
             self._binary_as_bytes = config.binary_as_bytes
@@ -620,6 +700,8 @@ def __init__(self, udf: Callable[..., Any], config):
             self._safecheck = config.arrow_convert_safely
 
     def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array:
+        if PYSPARK_VERSION >= (4, 2):
+            return _output_column(self._udf(None, iter([_record_batch(args, num_rows=num_rows)])))
         if self._use_legacy:
             return self._call_legacy(args, num_rows)
         return self._call_arrow(args, num_rows)
@@ -675,6 +757,7 @@ def __init__(
             struct_in_pandas="dict",
             ndarray_as_list=False,
             arrow_cast=False,
+            **_pandas_serializer_kwargs(config),
         )
 
     def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array:
@@ -698,6 +781,7 @@ def __init__(
             struct_in_pandas="dict",
             ndarray_as_list=False,
             arrow_cast=False,
+            **_pandas_serializer_kwargs(config),
         )
 
     def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array:
@@ -720,7 +804,9 @@ def __init__(
     ):
         self._udf = udf
 
-    def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array:
+    def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array:
+        if PYSPARK_VERSION >= (4, 2):
+            return _output_column(self._udf(None, iter([_record_batch(args, num_rows=num_rows)])))
         inputs = tuple(args)
         [(output, output_type)] = list(self._udf(None, (inputs,)))
         if isinstance(output, pa.ChunkedArray):
@@ -747,7 +833,9 @@ def __init__(
     ):
         self._udf = udf
 
-    def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array:
+    def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array:
+        if PYSPARK_VERSION >= (4, 2):
+            return _output_column(self._udf(None, iter([_record_batch(args, num_rows=num_rows)])))
         inputs = tuple(args)
         [(output, output_type)] = list(self._udf(None, [inputs]))
         if isinstance(output, pa.ChunkedArray):
@@ -777,9 +865,14 @@ def __init__(
             struct_in_pandas="dict",
             ndarray_as_list=False,
             arrow_cast=False,
+            **_pandas_serializer_kwargs(config),
         )
 
     def __call__(self, args: list[pa.Array]) -> pa.Array:
+        if PYSPARK_VERSION >= (4, 2):
+            inputs = _named_arrays_to_pandas(args, self._input_names, self._serializer)
+            [(output, output_type)] = list(self._udf(None, ([inputs],)))
+            return _pandas_to_arrow_array(output, output_type, self._serializer)
         inputs = _named_arrays_to_pandas(args, self._input_names, self._serializer)
         [(output, output_type)] = list(self._udf(None, (inputs,)))
         return _pandas_to_arrow_array(output, output_type, self._serializer)
@@ -800,6 +893,9 @@ def __init__(
         self._udf = udf
 
     def __call__(self, args: list[pa.Array]) -> pa.Array:
+        if PYSPARK_VERSION >= (4, 2):
+            batch = _record_batch(args)
+            return _output_column(self._udf(None, iter([iter([batch])])))
         [(output, output_type)] = list(self._udf(None, (args,)))
         if isinstance(output, pa.ChunkedArray):
             output = output.combine_chunks()
@@ -836,9 +932,24 @@ def __init__(
             struct_in_pandas="dict",
             ndarray_as_list=False,
             arrow_cast=False,
+            **_pandas_serializer_kwargs(config),
         )
 
     def __call__(self, args: list[pa.Array]) -> pa.Array:
+        if PYSPARK_VERSION >= (4, 2):
+            batch = pa.RecordBatch.from_arrays(args, self._input_names)
+            if self.is_pandas:
+                if self.is_iter:
+                    result = self._udf(None, iter([iter([batch])]))
+                    arrays = [
+                        _pandas_to_arrow_array(output, output_type, self._serializer)
+                        for group_outputs in result
+                        for outputs in group_outputs
+                        for output, output_type in outputs
+                    ]
+                    return pa.array([]) if not arrays else pa.concat_arrays(arrays)
+                return _output_column(self._udf(None, iter([iter([batch])])))
+            return _output_column(self._udf(None, iter([iter([_wrap_struct(batch)])])))
         m = self._max_records_per_batch
         if self.is_pandas:
             inputs = _named_arrays_to_pandas(args, self._input_names, self._serializer)
@@ -898,6 +1009,7 @@ def __init__(
             struct_in_pandas="dict",
             ndarray_as_list=False,
             arrow_cast=False,
+            **_pandas_serializer_kwargs(config),
         )
 
     def __call__(self, left: list[pa.Array], right: list[pa.Array]) -> pa.Array:
@@ -932,6 +1044,7 @@ def __init__(
             struct_in_pandas="dict",
             ndarray_as_list=False,
             arrow_cast=False,
+            **_pandas_serializer_kwargs(config),
         )
 
     def __call__(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
@@ -951,6 +1064,9 @@ def __init__(self, udf: Callable[..., Iterator[pa.RecordBatch]]):
         self._udf = udf
 
     def __call__(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
+        if PYSPARK_VERSION >= (4, 2):
+            output = self._udf(None, (_wrap_struct(batch) for batch in args))
+            return (_flatten_struct(batch) for batch in output)
         output = self._udf(None, ((x,) for x in args))
         return (x for x, _ in output)
 
@@ -1025,15 +1141,13 @@ def __init__(
         self._passthrough_columns = passthrough_columns
         self._output_schema = output_schema
         self._output_type = pa.struct([output_schema.field(i) for i in range(len(output_schema.names))])
-        self._use_legacy = config.python_udtf_pandas_conversion_enabled or pyspark.__version__.startswith(
-            ("3.", "4.0.")
-        )
+        self._use_legacy = config.python_udtf_pandas_conversion_enabled or (3,) <= PYSPARK_VERSION < (4, 1)
         if self._use_legacy:
-            if pyspark.__version__.startswith(("3.", "4.0.")):
+            if (3,) <= PYSPARK_VERSION < (4, 1):
                 self._serializer = ArrowStreamPandasUDTFSerializer(
                     timezone=config.session_timezone, safecheck=config.arrow_convert_safely
                 )
-            else:
+            elif (4, 1) <= PYSPARK_VERSION < (4, 2):
                 spark_input_types = [from_arrow_type(t) for t in input_types]
                 self._serializer = ArrowStreamPandasUDTFSerializer(
                     timezone=config.session_timezone,
@@ -1041,6 +1155,20 @@ def __init__(
                     input_types=spark_input_types,
                     int_to_decimal_coercion_enabled=config.python_udf_pandas_int_to_decimal_coercion_enabled,
                 )
+            else:
+                input_type = StructType(
+                    [
+                        StructField(name, from_arrow_type(data_type), True)
+                        for name, data_type in zip(input_names, input_types, strict=True)
+                    ]
+                )
+                self._serializer = ArrowStreamPandasUDTFSerializer(
+                    timezone=config.session_timezone,
+                    safecheck=config.arrow_convert_safely,
+                    input_type=input_type,
+                    prefer_int_ext_dtype=config.python_udf_pandas_prefer_int_extension_dtype,
+                    int_to_decimal_coercion_enabled=config.python_udf_pandas_int_to_decimal_coercion_enabled,
+                )
 
     def __call__(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
         if self._use_legacy:
@@ -1052,10 +1180,21 @@ def _call_arrow(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch
         if first is None:
             return
         args = itertools.chain([first], args)
-        # Tee: one branch for the mapper (full batches), one for passthrough extraction.
-        # The mapper expects full input batches — it handles Python conversion and column
-        # selection internally (via converters + args_kwargs_offsets).
+        # Tee: one branch for the mapper, one for passthrough extraction.
         batches_for_passthrough, batches_for_mapper = itertools.tee(args)
+        if PYSPARK_VERSION >= (4, 1):
+            # Spark 4.1+ interprets UDTF argument offsets relative to the input
+            # supplied to its worker. Sail's execution batches also include
+            # lateral-join passthrough columns, so remove those before invoking
+            # the worker and retain the original stream for result expansion.
+            def strip_passthrough(batch: pa.RecordBatch) -> pa.RecordBatch:
+                return _record_batch(
+                    batch.columns[self._passthrough_columns :],
+                    names=batch.schema.names[self._passthrough_columns :],
+                    num_rows=batch.num_rows,
+                )
+
+            batches_for_mapper = (strip_passthrough(batch) for batch in batches_for_mapper)
         outputs = self._udf(None, batches_for_mapper)
         # Extract passthrough values from raw arrow batches (no pandas)
         columns = (tuple(batch.column(i) for i in range(batch.num_columns)) for batch in batches_for_passthrough)
@@ -1094,7 +1233,7 @@ def _iter_input_legacy(self, args: Iterator[pa.RecordBatch]) -> Iterator[tuple[p
         for batch in args:
             arrays = batch.to_struct_array().flatten()
             columns = tuple(_arrow_column_to_pandas(x, self._serializer) for x in arrays)
-            if len(columns) == 0 and not pyspark.__version__.startswith(("3.", "4.0.")):
+            if len(columns) == 0 and PYSPARK_VERSION >= (4, 1):
                 # PySpark 4.1+ legacy mapper uses len(a[0]) for num_rows.
                 # Preserve the row count for 0-column batches (e.g. 0-arg UDTFs).
                 yield (pd.Series(range(batch.num_rows)),)
@@ -1106,12 +1245,9 @@ def _iter_output_legacy(self, args: Iterator[pa.RecordBatch]) -> Iterator[pd.Dat
         if sum(1 for _ in args2) == 0:
             return
         (batches1, batches2) = itertools.tee(self._iter_input_legacy(args1))
-        if pyspark.__version__.startswith(("3.", "4.0.")):
-            # PySpark 3.x/4.0 mapper receives sliced non-passthrough columns.
-            inputs = (x[self._passthrough_columns :] for x in batches2)
-        else:
-            # PySpark 4.1+ mapper receives full tuples and selects columns via args_kwargs_offsets.
-            inputs = batches2
+        # PySpark 3.x/4.0 mapper receives sliced non-passthrough columns.
+        # PySpark 4.1+ receives full tuples and selects columns via args_kwargs_offsets.
+        inputs = (x[self._passthrough_columns :] for x in batches2) if (3,) <= PYSPARK_VERSION < (4, 1) else batches2
         outputs = self._udf(None, inputs)
         last = None
         for passthrough, (out, *_) in itertools.zip_longest(self._iter_passthrough(batches1), outputs):
diff --git a/crates/sail-python-udf/src/python/spark.rs b/crates/sail-python-udf/src/python/spark.rs
index 8a724a91b..6ed52c40f 100644
--- a/crates/sail-python-udf/src/python/spark.rs
+++ b/crates/sail-python-udf/src/python/spark.rs
@@ -3,7 +3,7 @@ use std::ffi::CString;
 use datafusion::arrow::datatypes::{DataType, SchemaRef};
 use pyo3::prelude::{PyAnyMethods, PyModule};
 use pyo3::sync::PyOnceLock;
-use pyo3::{intern, Bound, Py, PyAny, PyResult, Python};
+use pyo3::{Bound, Py, PyAny, PyResult, Python, intern};
 
 use crate::config::PySparkUdfConfig;
 use crate::conversion::TryToPy;
@@ -38,12 +38,16 @@ impl PySpark {
         udf: Bound<'py, PyAny>,
         input_types: &[DataType],
         output_type: &DataType,
-        _config: &PySparkUdfConfig,
+        config: &PySparkUdfConfig,
     ) -> PyResult> {
         py_init_object(
             Self::module(py)?,
             intern!(py, "PySparkBatchUdf"),
-            (udf, input_types.try_to_py(py)?, output_type.try_to_py(py)?),
+            (
+                udf,
+                input_types.try_to_py(py, config.arrow_use_large_var_types)?,
+                output_type.try_to_py(py, config.arrow_use_large_var_types)?,
+            ),
         )
     }
 
@@ -202,9 +206,9 @@ impl PySpark {
             intern!(py, "PySparkTableUdf"),
             (
                 udf,
-                input_types.try_to_py(py)?,
+                input_types.try_to_py(py, config.arrow_use_large_var_types)?,
                 passthrough_columns,
-                output_schema.try_to_py(py)?,
+                output_schema.try_to_py(py, config.arrow_use_large_var_types)?,
                 config.clone(),
             ),
         )
@@ -225,9 +229,9 @@ impl PySpark {
             (
                 udf,
                 input_names.to_vec(),
-                input_types.try_to_py(py)?,
+                input_types.try_to_py(py, config.arrow_use_large_var_types)?,
                 passthrough_columns,
-                output_schema.try_to_py(py)?,
+                output_schema.try_to_py(py, config.arrow_use_large_var_types)?,
                 config.clone(),
             ),
         )
diff --git a/crates/sail-python-udf/src/stream.rs b/crates/sail-python-udf/src/stream.rs
index 4aa67f107..1c6a59de2 100644
--- a/crates/sail-python-udf/src/stream.rs
+++ b/crates/sail-python-udf/src/stream.rs
@@ -6,18 +6,19 @@ use datafusion::arrow::array::RecordBatch;
 use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use datafusion_common::{exec_err, DataFusionError, Result};
+use datafusion_common::{DataFusionError, Result, exec_err};
 use futures::{Stream, StreamExt};
 use pyo3::exceptions::{PyRuntimeError, PyStopIteration};
 use pyo3::prelude::PyAnyMethods;
-use pyo3::{pyclass, pymethods, IntoPyObject, Py, PyAny, PyRef, PyRefMut, PyResult, Python};
+use pyo3::{IntoPyObject, Py, PyAny, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods};
 use sail_common_datafusion::array::record_batch::cast_record_batch_positionally;
-use sail_pyarrow::{FromPyArrow, ToPyArrow};
+use sail_pyarrow::FromPyArrow;
 use tokio::runtime::Handle;
 use tokio::select;
 use tokio::sync::{mpsc, oneshot};
 use tokio_stream::wrappers::ReceiverStream;
 
+use crate::conversion::TryToPy;
 use crate::error::PyUdfResult;
 
 struct PyInputStreamState {
@@ -38,6 +39,7 @@ impl PyInputStreamState {
 struct PyInputStream {
     state: Arc>,
     handle: Handle,
+    large_var_types: bool,
 }
 
 impl PyInputStream {
@@ -45,6 +47,7 @@ impl PyInputStream {
         input: SendableRecordBatchStream,
         signal: oneshot::Receiver<()>,
         handle: Handle,
+        large_var_types: bool,
     ) -> Self {
         Self {
             state: Arc::new(tokio::sync::Mutex::new(PyInputStreamState {
@@ -52,6 +55,7 @@ impl PyInputStream {
                 signal,
             })),
             handle,
+            large_var_types,
         }
     }
 }
@@ -65,6 +69,7 @@ impl PyInputStream {
     fn __next__(self_: PyRefMut<'_, Self>) -> PyResult> {
         let state = Arc::clone(&self_.state);
         let handle = self_.handle.clone();
+        let large_var_types = self_.large_var_types;
         let py = self_.py();
         py.detach(|| {
             handle.block_on(async {
@@ -76,7 +81,7 @@ impl PyInputStream {
                     .map(|x| x.map_err(|e| PyRuntimeError::new_err(e.to_string())))
             })
         })
-        .map(|x| x.and_then(|x| x.to_pyarrow(py).map(|b| b.unbind())))
+        .map(|x| x.and_then(|x| x.try_to_py(py, large_var_types).map(|batch| batch.unbind())))
         .unwrap_or(Err(PyStopIteration::new_err("")))
     }
 }
@@ -101,6 +106,7 @@ impl PyMapStream {
         input: SendableRecordBatchStream,
         function: Py,
         output_schema: SchemaRef,
+        large_var_types: bool,
     ) -> Self {
         let (output_tx, output_rx) = mpsc::channel(Self::OUTPUT_CHANNEL_BUFFER);
         let (signal_tx, signal_rx) = oneshot::channel();
@@ -118,6 +124,7 @@ impl PyMapStream {
                     signal_rx,
                     output_tx.clone(),
                     handle,
+                    large_var_types,
                 )
             }) {
                 Ok(()) => {}
@@ -146,12 +153,13 @@ impl PyMapStream {
         signal: oneshot::Receiver<()>,
         sender: mpsc::Sender>,
         handle: Handle,
+        large_var_types: bool,
     ) -> PyUdfResult<()> {
         // Create a Python iterator from the input record batch stream and call the function.
         // We could have wrap each record batch in a single-element list and call the function
         // for each record batch, but that does not work if the user wants to maintain state
         // across record batches.
-        let input = PyInputStream::new(input, signal, handle);
+        let input = PyInputStream::new(input, signal, handle, large_var_types);
         let input = input.into_pyobject(py)?;
         let output = function.call1(py, (input,))?.into_bound(py);
         for batch in output.try_iter()? {
diff --git a/crates/sail-python-udf/src/udf/pyspark_batch_collector.rs b/crates/sail-python-udf/src/udf/pyspark_batch_collector.rs
index e313b5b64..bb874ccef 100644
--- a/crates/sail-python-udf/src/udf/pyspark_batch_collector.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_batch_collector.rs
@@ -4,8 +4,8 @@ use datafusion::arrow::array::{ArrayRef, StructArray};
 use datafusion::arrow::datatypes::{DataType, Field, FieldRef};
 use datafusion::logical_expr::{Accumulator, Signature, Volatility};
 use datafusion_common::Result;
-use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
 use datafusion_expr::AggregateUDFImpl;
+use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
 
 use crate::accumulator::{BatchAggregateAccumulator, BatchAggregator};
 use crate::array::{build_singleton_list_array, get_fields, get_struct_array_type};
diff --git a/crates/sail-python-udf/src/udf/pyspark_cogroup_map_udf.rs b/crates/sail-python-udf/src/udf/pyspark_cogroup_map_udf.rs
index d2030e5af..48a0ca8d6 100644
--- a/crates/sail-python-udf/src/udf/pyspark_cogroup_map_udf.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_cogroup_map_udf.rs
@@ -174,7 +174,13 @@ impl ScalarUDFImpl for PySparkCoGroupMapUDF {
                 let left = Self::get_group(&left, i)?;
                 let right = Self::get_group(&right, i)?;
                 let data = Python::attach(|py| -> PyUdfResult<_> {
-                    let output = udf.call1(py, (left.try_to_py(py)?, right.try_to_py(py)?))?;
+                    let output = udf.call1(
+                        py,
+                        (
+                            left.try_to_py(py, self.config.arrow_use_large_var_types)?,
+                            right.try_to_py(py, self.config.arrow_use_large_var_types)?,
+                        ),
+                    )?;
                     Ok(ArrayData::try_from_py(py, &output)?)
                 })?;
                 let array = cast(&make_array(data), field.data_type())?;
diff --git a/crates/sail-python-udf/src/udf/pyspark_group_map_udf.rs b/crates/sail-python-udf/src/udf/pyspark_group_map_udf.rs
index 6cab5f349..bb43592b4 100644
--- a/crates/sail-python-udf/src/udf/pyspark_group_map_udf.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_group_map_udf.rs
@@ -1,12 +1,12 @@
 use std::sync::Arc;
 
-use datafusion::arrow::array::{make_array, ArrayData, ArrayRef};
+use datafusion::arrow::array::{ArrayData, ArrayRef, make_array};
 use datafusion::arrow::compute::cast;
 use datafusion::arrow::datatypes::{DataType, FieldRef};
 use datafusion::logical_expr::{Accumulator, Signature, Volatility};
 use datafusion_common::Result;
-use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
 use datafusion_expr::AggregateUDFImpl;
+use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
 use pyo3::{Py, PyAny, Python};
 
 use crate::accumulator::{BatchAggregateAccumulator, BatchAggregator};
@@ -138,7 +138,11 @@ impl AggregateUDFImpl for PySparkGroupMapUDF {
     fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result> {
         let field = get_list_field(&self.output_type)?;
         let udf = Python::attach(|py| self.udf(py))?;
-        let aggregator = Box::new(PySparkGroupMapper { udf, field });
+        let aggregator = Box::new(PySparkGroupMapper {
+            udf,
+            field,
+            large_var_types: self.config.arrow_use_large_var_types,
+        });
         Ok(Box::new(BatchAggregateAccumulator::new(
             self.input_types.clone(),
             self.output_type.clone(),
@@ -155,12 +159,15 @@ impl AggregateUDFImpl for PySparkGroupMapUDF {
 struct PySparkGroupMapper {
     udf: Py,
     field: FieldRef,
+    large_var_types: bool,
 }
 
 impl BatchAggregator for PySparkGroupMapper {
     fn call(&self, args: &[ArrayRef]) -> Result {
         let data = Python::attach(|py| -> PyUdfResult<_> {
-            let output = self.udf.call1(py, (args.try_to_py(py)?,))?;
+            let output = self
+                .udf
+                .call1(py, (args.try_to_py(py, self.large_var_types)?,))?;
             Ok(ArrayData::try_from_py(py, &output)?)
         })?;
         let array = cast(&make_array(data), self.field.data_type())?;
diff --git a/crates/sail-python-udf/src/udf/pyspark_map_iter_udf.rs b/crates/sail-python-udf/src/udf/pyspark_map_iter_udf.rs
index 7336d74d8..e82c0bf45 100644
--- a/crates/sail-python-udf/src/udf/pyspark_map_iter_udf.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_map_iter_udf.rs
@@ -90,6 +90,7 @@ impl StreamUDF for PySparkMapIterUDF {
             rename_record_batch_stream(input, &self.input_names)?,
             function,
             self.output_schema.clone(),
+            self.config.arrow_use_large_var_types,
         )))
     }
 }
diff --git a/crates/sail-python-udf/src/udf/pyspark_udaf.rs b/crates/sail-python-udf/src/udf/pyspark_udaf.rs
index 792b41fbb..8d6167ac5 100644
--- a/crates/sail-python-udf/src/udf/pyspark_udaf.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_udaf.rs
@@ -1,13 +1,13 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
-use datafusion::arrow::array::{make_array, ArrayData, ArrayRef};
+use datafusion::arrow::array::{ArrayData, ArrayRef, make_array};
 use datafusion::arrow::compute::cast;
 use datafusion::arrow::datatypes::{DataType, FieldRef};
 use datafusion::common::Result;
 use datafusion::logical_expr::{Accumulator, Signature, Volatility};
-use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
 use datafusion_expr::AggregateUDFImpl;
+use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
 use pyo3::{Py, PyAny, Python};
 
 use crate::accumulator::{BatchAggregateAccumulator, BatchAggregator};
@@ -147,6 +147,7 @@ impl AggregateUDFImpl for PySparkGroupAggregateUDF {
         let aggregator = Box::new(PySparkGroupAggregator {
             udf,
             output_type: self.output_type.clone(),
+            large_var_types: self.config.arrow_use_large_var_types,
         });
         Ok(Box::new(BatchAggregateAccumulator::new(
             self.input_types.clone(),
@@ -164,12 +165,15 @@ impl AggregateUDFImpl for PySparkGroupAggregateUDF {
 struct PySparkGroupAggregator {
     udf: Py,
     output_type: DataType,
+    large_var_types: bool,
 }
 
 impl BatchAggregator for PySparkGroupAggregator {
     fn call(&self, args: &[ArrayRef]) -> Result {
         let data = Python::attach(|py| -> PyUdfResult<_> {
-            let output = self.udf.call1(py, (args.try_to_py(py)?,))?;
+            let output = self
+                .udf
+                .call1(py, (args.try_to_py(py, self.large_var_types)?,))?;
             Ok(ArrayData::try_from_py(py, &output)?)
         })?;
         let array = cast(&make_array(data), &self.output_type)?;
diff --git a/crates/sail-python-udf/src/udf/pyspark_udf.rs b/crates/sail-python-udf/src/udf/pyspark_udf.rs
index 32f1ba612..8004d997a 100644
--- a/crates/sail-python-udf/src/udf/pyspark_udf.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_udf.rs
@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
-use datafusion::arrow::array::{make_array, ArrayData, ArrayRef};
+use datafusion::arrow::array::{ArrayData, ArrayRef, make_array};
 use datafusion::arrow::compute::cast;
 use datafusion::arrow::datatypes::DataType;
 use datafusion::common::Result;
@@ -136,7 +136,13 @@ impl ScalarUDFImpl for PySparkUDF {
         let args: Vec = ColumnarValue::values_to_arrays(&args)?;
         let udf = Python::attach(|py| self.udf(py))?;
         let data = Python::attach(|py| -> PyUdfResult<_> {
-            let output = udf.call1(py, (args.try_to_py(py)?, number_rows))?;
+            let output = udf.call1(
+                py,
+                (
+                    args.try_to_py(py, self.config.arrow_use_large_var_types)?,
+                    number_rows,
+                ),
+            )?;
             Ok(ArrayData::try_from_py(py, &output)?)
         })?;
         let array = cast(&make_array(data), &self.output_type)?;
diff --git a/crates/sail-python-udf/src/udf/pyspark_udtf.rs b/crates/sail-python-udf/src/udf/pyspark_udtf.rs
index c8fa87a2c..f5cc69a28 100644
--- a/crates/sail-python-udf/src/udf/pyspark_udtf.rs
+++ b/crates/sail-python-udf/src/udf/pyspark_udtf.rs
@@ -209,6 +209,7 @@ impl StreamUDF for PySparkUDTF {
             input,
             function,
             self.output_schema.clone(),
+            self.config.arrow_use_large_var_types,
         )))
     }
 }
diff --git a/crates/sail-python/src/cli.rs b/crates/sail-python/src/cli.rs
index 530837a3a..3ddf33f26 100644
--- a/crates/sail-python/src/cli.rs
+++ b/crates/sail-python/src/cli.rs
@@ -1,5 +1,5 @@
 use pyo3::exceptions::PyRuntimeError;
-use pyo3::{pyfunction, PyErr, PyResult, Python};
+use pyo3::{PyErr, PyResult, Python, pyfunction};
 
 /// The Sail CLI entrypoint exposed to Python.
 #[pyfunction]
diff --git a/crates/sail-python/src/flight/server.rs b/crates/sail-python/src/flight/server.rs
index a68ad0d43..24504afc2 100644
--- a/crates/sail-python/src/flight/server.rs
+++ b/crates/sail-python/src/flight/server.rs
@@ -8,7 +8,7 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError};
 use pyo3::prelude::*;
 use sail_common::config::AppConfig;
 use sail_common::runtime::RuntimeHandle;
-use sail_flight::serve;
+use sail_flight::entrypoint::serve;
 use tokio::net::TcpListener;
 use tokio::runtime::Handle;
 use tokio::sync::oneshot::{Receiver, Sender};
diff --git a/crates/sail-python/src/globals.rs b/crates/sail-python/src/globals.rs
index 0a9762b0e..573b85436 100644
--- a/crates/sail-python/src/globals.rs
+++ b/crates/sail-python/src/globals.rs
@@ -8,7 +8,7 @@ use pyo3::prelude::*;
 use pyo3::sync::PyOnceLock;
 use sail_common::config::{AppConfig, SAIL_ENV_VAR_PREFIX};
 use sail_common::runtime::RuntimeManager;
-use sail_telemetry::telemetry::{init_telemetry, ResourceOptions};
+use sail_telemetry::telemetry::{ResourceOptions, init_telemetry};
 
 static GLOBALS: PyOnceLock = PyOnceLock::new();
 
diff --git a/crates/sail-server/Cargo.toml b/crates/sail-server/Cargo.toml
deleted file mode 100644
index 31678231c..000000000
--- a/crates/sail-server/Cargo.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-[package]
-name = "sail-server"
-version = { workspace = true }
-edition = { workspace = true }
-
-[lints]
-workspace = true
-
-[dependencies]
-sail-common = { path = "../sail-common" }
-sail-telemetry = { path = "../sail-telemetry" }
-
-tokio = { workspace = true }
-tonic = { workspace = true }
-tonic-reflection = { workspace = true }
-tonic-health = { workspace = true }
-fastrace = { workspace = true }
-tower = { workspace = true }
-log = { workspace = true }
-axum = { workspace = true }
diff --git a/crates/sail-session/Cargo.toml b/crates/sail-session/Cargo.toml
index 5c8d49526..fe2f20fba 100644
--- a/crates/sail-session/Cargo.toml
+++ b/crates/sail-session/Cargo.toml
@@ -27,9 +27,7 @@ sail-physical-optimizer = { path = "../sail-physical-optimizer" }
 sail-plan = { path = "../sail-plan" }
 sail-execution = { path = "../sail-execution" }
 sail-object-store = { path = "../sail-object-store" }
-sail-server = { path = "../sail-server" }
 sail-cache = { path = "../sail-cache" }
-sail-telemetry = { path = "../sail-telemetry" }
 sail-sql-analyzer = { path = "../sail-sql-analyzer" }
 
 async-trait = { workspace = true }
@@ -39,11 +37,17 @@ fastrace = { workspace = true }
 log = { workspace = true }
 datafusion = { workspace = true }
 datafusion-common = { workspace = true }
+datafusion-datasource = { workspace = true }
 datafusion-expr = { workspace = true }
 datafusion-physical-expr = { workspace = true }
+object_store = { workspace = true }
 secrecy = { workspace = true }
 tonic = { workspace = true }
 tokio = { workspace = true }
 chrono = { workspace = true }
 indexmap = { workspace = true }
 futures = { workspace = true }
+uuid = { workspace = true }
+
+[dev-dependencies]
+tempfile = { workspace = true }
diff --git a/crates/sail-session/src/catalog.rs b/crates/sail-session/src/catalog.rs
index 58975ac6b..b147b4c68 100644
--- a/crates/sail-session/src/catalog.rs
+++ b/crates/sail-session/src/catalog.rs
@@ -1,8 +1,11 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use datafusion::common::{plan_datafusion_err, Result};
+use datafusion::common::{Result, plan_datafusion_err};
 use datafusion_common::plan_err;
+use sail_catalog::credentials::{
+    CatalogCredentials, EmptyCatalogCredentials, FileCatalogCredentials, StaticCatalogCredentials,
+};
 use sail_catalog::error::CatalogResult;
 use sail_catalog::manager::{CatalogManager, CatalogManagerOptions};
 use sail_catalog::provider::{
@@ -10,14 +13,14 @@ use sail_catalog::provider::{
 };
 use sail_catalog_glue::{GlueCatalogConfig, GlueCatalogProvider};
 use sail_catalog_hms::{HmsCatalogConfig, HmsCatalogProvider};
-use sail_catalog_iceberg::IcebergRestCatalogProvider;
+use sail_catalog_iceberg::{IcebergRestCatalogOptions, IcebergRestCatalogProvider};
 use sail_catalog_memory::MemoryCatalogProvider;
-use sail_catalog_onelake::OneLakeCatalogProvider;
-use sail_catalog_system::{SystemCatalogProvider, SYSTEM_CATALOG_NAME};
-use sail_catalog_unity::UnityCatalogProvider;
-use sail_common::config::{AppConfig, CacheType, CatalogCacheConfig, CatalogType};
+use sail_catalog_onelake::{OneLakeApiKind, OneLakeCatalogProvider};
+use sail_catalog_system::{SYSTEM_CATALOG_NAME, SystemCatalogProvider};
+use sail_catalog_unity::{UnityCatalogConfig, UnityCatalogOptions, UnityCatalogProvider};
+use sail_common::config::{AppConfig, CacheType, CatalogCacheConfig, CatalogType, OneLakeApi};
 use sail_common::runtime::RuntimeHandle;
-use secrecy::ExposeSecret;
+use secrecy::{ExposeSecret, SecretString};
 
 pub fn create_catalog_manager(
     config: &AppConfig,
@@ -50,6 +53,7 @@ pub fn create_catalog_manager(
                     namespace_separator,
                     oauth_access_token,
                     bearer_access_token,
+                    bearer_access_token_file,
                     cache,
                 } => {
                     let mut properties = HashMap::new();
@@ -66,23 +70,21 @@ pub fn create_catalog_manager(
                             namespace_separator.to_string(),
                         );
                     }
-                    if let Some(oauth_access_token) = oauth_access_token {
-                        properties.insert(
-                            "oauth-access-token".to_string(), // Iceberg uses kebab-case
-                            oauth_access_token.expose_secret().to_string(), // FIXME: Only expose when necessary
-                        );
-                    }
-                    if let Some(bearer_access_token) = bearer_access_token {
-                        properties.insert(
-                            "bearer-access-token".to_string(), // Iceberg uses kebab-case
-                            bearer_access_token.expose_secret().to_string(), // FIXME: Only expose when necessary
-                        );
-                    }
+                    let credentials = iceberg_rest_credentials(
+                        bearer_access_token_file.as_ref(),
+                        bearer_access_token.as_ref(),
+                        oauth_access_token.as_ref(),
+                    );
 
                     let runtime_aware = RuntimeAwareCatalogProvider::try_new(
                         || {
-                            let provider =
-                                IcebergRestCatalogProvider::new(name.to_string(), properties);
+                            let provider = IcebergRestCatalogProvider::new(
+                                name.to_string(),
+                                IcebergRestCatalogOptions {
+                                    credentials,
+                                    properties,
+                                },
+                            );
                             Ok(provider)
                         },
                         runtime.io().clone(),
@@ -103,7 +105,27 @@ pub fn create_catalog_manager(
                     cache,
                 } => {
                     let runtime_aware = RuntimeAwareCatalogProvider::try_new(
-                        || UnityCatalogProvider::new(name.to_string(), default_catalog, uri, token),
+                        || {
+                            let config = UnityCatalogConfig::new(uri.clone(), token, None)?;
+                            let credentials = config
+                                .get_credential_provider()
+                                .map(|credentials| {
+                                    Arc::new(credentials) as Arc
+                                })
+                                .unwrap_or_else(|| Arc::new(EmptyCatalogCredentials));
+                            let default_catalog = default_catalog
+                                .clone()
+                                .unwrap_or_else(|| "unity".to_string());
+                            UnityCatalogProvider::new(
+                                name.to_string(),
+                                UnityCatalogOptions {
+                                    default_catalog,
+                                    uri: config.uri,
+                                    credentials,
+                                    quote_object_name: true,
+                                },
+                            )
+                        },
                         runtime.io().clone(),
                     )?;
                     let provider = wrap_catalog_provider(
@@ -117,35 +139,44 @@ pub fn create_catalog_manager(
                 CatalogType::OneLake {
                     name,
                     url,
+                    api,
                     bearer_token,
                     cache,
                 } => {
-                    // Parse URL format: workspace/item.type (e.g., "duckrun/data.lakehouse", "duckrun/data.datawarehouse")
+                    // Parse URL format:
+                    //   - `workspace/item.type`: friendly names (e.g. "duckrun/data.lakehouse")
+                    //   - `workspaceId/itemId`: GUIDs (e.g. "8f.../3a...").
+                    // GUID form is required when the workspace has friendly-name support disabled,
+                    // because the data/metadata paths are then GUID-addressed at the storage layer.
                     let (workspace, item) = url.split_once('/').ok_or_else(|| {
                         plan_datafusion_err!(
-                            "Invalid OneLake URL format: expected 'workspace/item.type', got '{}'",
+                            "Invalid OneLake URL format: expected 'workspace/item.type' or 'workspaceId/itemId', got '{}'",
                             url
                         )
                     })?;
 
-                    // Extract item name and type (e.g., "data.Lakehouse" -> name="data", type="Lakehouse")
-                    let (item_name, item_type) = item.split_once('.').ok_or_else(|| {
-                        plan_datafusion_err!(
-                            "Invalid OneLake item format: expected 'name.type', got '{}'",
-                            item
-                        )
-                    })?;
+                    // Friendly form: ("data.Lakehouse" -> name="data", type=Some("Lakehouse"))
+                    // GUID form: ("data" -> name="data", type=None)
+                    let (item_name, item_type) = match item.split_once('.') {
+                        Some((name, item_type)) => (name, Some(item_type.to_string())),
+                        None => (item, None),
+                    };
 
                     let token = bearer_token.as_ref().map(|t| t.expose_secret().to_string());
+                    let api = match api {
+                        OneLakeApi::Delta => OneLakeApiKind::Delta,
+                        OneLakeApi::Iceberg => OneLakeApiKind::Iceberg,
+                    };
                     let runtime_aware = RuntimeAwareCatalogProvider::try_new(
                         || {
-                            Ok(OneLakeCatalogProvider::new(
+                            OneLakeCatalogProvider::new(
                                 name.clone(),
                                 workspace.to_string(),
                                 item_name.to_string(),
-                                item_type.to_string(),
+                                item_type,
+                                api,
                                 token.clone(),
-                            ))
+                            )
                         },
                         runtime.io().clone(),
                     )?;
@@ -159,11 +190,13 @@ pub fn create_catalog_manager(
                 }
                 CatalogType::Glue {
                     name,
+                    catalog_id,
                     region,
                     endpoint_url,
                     cache,
                 } => {
                     let config = GlueCatalogConfig {
+                        catalog_id: catalog_id.clone(),
                         region: region.clone(),
                         endpoint_url: endpoint_url.clone(),
                     };
@@ -281,6 +314,28 @@ fn wrap_catalog_provider(
     Ok(Arc::new(provider))
 }
 
+/// Credentials for an Iceberg REST catalog. A token file takes precedence over
+/// a static token: its contents are re-read per request, so a rotated projected
+/// service account token is picked up without a restart.
+fn iceberg_rest_credentials(
+    bearer_access_token_file: Option<&String>,
+    bearer_access_token: Option<&SecretString>,
+    oauth_access_token: Option<&SecretString>,
+) -> Arc {
+    if let Some(path) = bearer_access_token_file {
+        Arc::new(FileCatalogCredentials::new(path)) as Arc
+    } else {
+        bearer_access_token
+            .or(oauth_access_token)
+            .map(|token| {
+                Arc::new(StaticCatalogCredentials::new(
+                    token.expose_secret().to_string(),
+                )) as Arc
+            })
+            .unwrap_or_else(|| Arc::new(EmptyCatalogCredentials))
+    }
+}
+
 #[cfg(test)]
 #[expect(clippy::unwrap_used, clippy::expect_used)]
 mod tests {
@@ -316,6 +371,7 @@ mod tests {
         let mut config = AppConfig::load().unwrap();
         config.catalog.list = vec![CatalogType::Glue {
             name: "glue".to_string(),
+            catalog_id: None,
             region: None,
             endpoint_url: None,
             cache: CatalogCacheConfig {
@@ -351,6 +407,7 @@ mod tests {
         let mut config = AppConfig::load().unwrap();
         config.catalog.list = vec![CatalogType::Glue {
             name: "glue".to_string(),
+            catalog_id: None,
             region: None,
             endpoint_url: None,
             cache: CatalogCacheConfig {
@@ -374,4 +431,45 @@ mod tests {
             "session cache should not be stored in global manager"
         );
     }
+
+    #[tokio::test]
+    async fn iceberg_credentials_prefer_the_token_file() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let path = dir.path().join("token");
+        std::fs::write(&path, "file-token\n").unwrap();
+        let path = path.to_string_lossy().to_string();
+        let static_token = SecretString::from("static-token".to_string());
+
+        let credentials = iceberg_rest_credentials(Some(&path), Some(&static_token), None);
+        assert_eq!(
+            credentials.retrieve().await.unwrap(),
+            Some("file-token".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn iceberg_credentials_use_the_static_bearer_token() {
+        let token = SecretString::from("static-token".to_string());
+        let credentials = iceberg_rest_credentials(None, Some(&token), None);
+        assert_eq!(
+            credentials.retrieve().await.unwrap(),
+            Some("static-token".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn iceberg_credentials_fall_back_to_the_oauth_token() {
+        let token = SecretString::from("oauth-token".to_string());
+        let credentials = iceberg_rest_credentials(None, None, Some(&token));
+        assert_eq!(
+            credentials.retrieve().await.unwrap(),
+            Some("oauth-token".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn iceberg_credentials_are_empty_without_any_token() {
+        let credentials = iceberg_rest_credentials(None, None, None);
+        assert_eq!(credentials.retrieve().await.unwrap(), None);
+    }
 }
diff --git a/crates/sail-session/src/formats.rs b/crates/sail-session/src/formats.rs
index a71ad3533..daa9bd558 100644
--- a/crates/sail-session/src/formats.rs
+++ b/crates/sail-session/src/formats.rs
@@ -10,7 +10,7 @@ use sail_data_source::formats::csv::CsvTableFormat;
 use sail_data_source::formats::json::JsonTableFormat;
 use sail_data_source::formats::noop::NoopTableFormat;
 use sail_data_source::formats::parquet::ParquetTableFormat;
-use sail_data_source::formats::python::{discover_data_sources, PythonTableFormat};
+use sail_data_source::formats::python::{PythonTableFormat, discover_data_sources};
 use sail_data_source::formats::rate::RateTableFormat;
 use sail_data_source::formats::socket::SocketTableFormat;
 use sail_data_source::formats::text::TextTableFormat;
diff --git a/crates/sail-session/src/observable.rs b/crates/sail-session/src/observable.rs
index e1dca18c8..a42c14056 100644
--- a/crates/sail-session/src/observable.rs
+++ b/crates/sail-session/src/observable.rs
@@ -1,9 +1,9 @@
 use async_trait::async_trait;
 use datafusion_common::exec_datafusion_err;
+use sail_common::actor::ActorHandle;
 use sail_common_datafusion::system::observable::{
     Observer, SessionManagerObserver, StateObservable,
 };
-use sail_server::actor::ActorHandle;
 use tokio::sync::mpsc::error::SendError;
 
 use crate::session_manager::{SessionManagerActor, SessionManagerEvent};
diff --git a/crates/sail-session/src/optimizer.rs b/crates/sail-session/src/optimizer.rs
index e8a5e04df..fced53b72 100644
--- a/crates/sail-session/src/optimizer.rs
+++ b/crates/sail-session/src/optimizer.rs
@@ -21,8 +21,10 @@ mod tests {
     #[test]
     fn default_rules_skip_leaf_projection_pushdown() {
         let rules = default_optimizer_rules();
-        assert!(!rules
-            .iter()
-            .any(|rule| rule.name() == "push_down_leaf_projections"));
+        assert!(
+            !rules
+                .iter()
+                .any(|rule| rule.name() == "push_down_leaf_projections")
+        );
     }
 }
diff --git a/crates/sail-session/src/planner.rs b/crates/sail-session/src/planner.rs
index 02cd978a4..31b75d888 100644
--- a/crates/sail-session/src/planner.rs
+++ b/crates/sail-session/src/planner.rs
@@ -1,16 +1,33 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use datafusion::execution::context::QueryPlanner;
+use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions};
+use datafusion::common::config::TableParquetOptions;
+use datafusion::datasource::memory::MemorySourceConfig;
+use datafusion::datasource::physical_plan::ParquetSource;
 use datafusion::execution::SessionState;
-use datafusion::physical_expr::LexOrdering;
+use datafusion::execution::context::QueryPlanner;
+use datafusion::physical_expr::expressions::Column;
+use datafusion::physical_expr::{
+    LexOrdering, OrderingRequirements, PhysicalExpr, PhysicalSortExpr,
+};
+use datafusion::physical_optimizer::output_requirements::OutputRequirementExec;
 use datafusion::physical_plan::sorts::sort::SortExec;
-use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties};
 use datafusion::physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner};
-use datafusion_common::{internal_err, DFSchema};
+use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::{DFSchema, Statistics, internal_datafusion_err, internal_err};
+use datafusion_datasource::file_groups::FileGroup;
+use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
+use datafusion_datasource::source::{DataSource, DataSourceExec};
+use datafusion_datasource::{PartitionedFile, TableSchema};
 use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNode};
-use datafusion_physical_expr::{create_physical_sort_exprs, Partitioning};
+use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
+use datafusion_physical_expr::{Partitioning, create_physical_sort_exprs};
+use sail_cache::remote_checkpoint::RemoteCheckpointRegistry;
 use sail_catalog_system::planner::SystemTablePhysicalPlanner;
+use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::logical_rewriter::LogicalRewriter;
 use sail_common_datafusion::rename::physical_plan::rename_projected_physical_plan;
 use sail_common_datafusion::streaming::event::schema::{
@@ -26,10 +43,13 @@ use sail_logical_plan::barrier::BarrierNode;
 use sail_logical_plan::map_partitions::MapPartitionsNode;
 use sail_logical_plan::monotonic_id::MonotonicIdNode;
 use sail_logical_plan::range::RangeNode;
+use sail_logical_plan::remote_checkpoint::{
+    RemoteCheckpointCommandNode, RemoteCheckpointRelationNode,
+};
 use sail_logical_plan::repartition::{ExplicitRepartitionKind, ExplicitRepartitionNode};
 use sail_logical_plan::schema_pivot::SchemaPivotNode;
 use sail_logical_plan::show_string::ShowStringNode;
-use sail_logical_plan::sort::SortWithinPartitionsNode;
+use sail_logical_plan::sort::{RequiredSortNode, SortWithinPartitionsNode};
 use sail_logical_plan::spark_partition_id::SparkPartitionIdNode;
 use sail_logical_plan::streaming::collector::StreamCollectorNode;
 use sail_logical_plan::streaming::filter::StreamFilterNode;
@@ -41,6 +61,10 @@ use sail_physical_plan::catalog_command::CatalogCommandExec;
 use sail_physical_plan::map_partitions::MapPartitionsExec;
 use sail_physical_plan::monotonic_id::MonotonicIdExec;
 use sail_physical_plan::range::RangeExec;
+use sail_physical_plan::remote_checkpoint::{
+    CheckpointDataSource, RemoteCheckpointCommitExec, RemoteCheckpointWriteExec,
+    checkpoint_storage_schema,
+};
 use sail_physical_plan::repartition::ExplicitRepartitionExec;
 use sail_physical_plan::schema_pivot::SchemaPivotExec;
 use sail_physical_plan::show_string::ShowStringExec;
@@ -98,8 +122,193 @@ impl ExtensionPlanner for ExtensionPhysicalPlanner {
         session_state: &SessionState,
     ) -> datafusion_common::Result>> {
         let plan: Arc = if let Some(node) =
-            node.as_any().downcast_ref::()
+            node.as_any().downcast_ref::()
         {
+            let [_logical_input] = logical_inputs else {
+                return internal_err!("RemoteCheckpointCommand requires exactly one logical input");
+            };
+            let [input] = physical_inputs else {
+                return internal_err!(
+                    "RemoteCheckpointCommand requires exactly one physical input"
+                );
+            };
+            let registry = session_state.extension::()?;
+            let (object_store_url, prefix) = registry
+                .resolve_relation(session_state.runtime_env().as_ref(), node.relation_id())?;
+            let storage_schema = checkpoint_storage_schema(node.logical_schema());
+            let output_partitioning =
+                checkpoint_schema_partitioning(input.output_partitioning(), &storage_schema)?;
+            let output_ordering = input
+                .output_ordering()
+                .map(|ordering| checkpoint_schema_ordering(ordering, &storage_schema))
+                .transpose()?;
+            let writer: Arc = Arc::new(RemoteCheckpointWriteExec::try_new(
+                Arc::clone(input),
+                object_store_url.clone(),
+                prefix.clone(),
+                Arc::clone(&storage_schema),
+            )?);
+            Arc::new(RemoteCheckpointCommitExec::new(
+                writer,
+                node.relation_id().to_string(),
+                object_store_url,
+                prefix,
+                Arc::clone(node.logical_schema()),
+                storage_schema,
+                output_partitioning,
+                output_ordering,
+            ))
+        } else if let Some(node) = node.as_any().downcast_ref::() {
+            let registry = session_state.extension::()?;
+            let descriptor = registry.get(node.relation_id())?.ok_or_else(|| {
+                datafusion_common::DataFusionError::Plan(format!(
+                    "checkpoint relation is not available: {}",
+                    node.relation_id()
+                ))
+            })?;
+            let logical_schema = UserDefinedLogicalNode::schema(node).inner().clone();
+            let schema_matches = descriptor.storage_schema.fields().len()
+                == logical_schema.fields().len()
+                && descriptor
+                    .storage_schema
+                    .fields()
+                    .iter()
+                    .zip(logical_schema.fields())
+                    .all(|(storage, logical)| storage.data_type() == logical.data_type());
+            if !schema_matches {
+                return internal_err!(
+                    "checkpoint storage schema does not match the relation schema by position"
+                );
+            }
+            let output_partitioning = checkpoint_schema_partitioning(
+                &descriptor.output_partitioning,
+                logical_schema.as_ref(),
+            )?;
+            let storage_output_ordering = descriptor
+                .output_ordering
+                .as_ref()
+                .map(|ordering| {
+                    checkpoint_schema_ordering(ordering, descriptor.storage_schema.as_ref())
+                })
+                .transpose()?;
+            let scan: Arc = if descriptor.storage_schema.fields().is_empty() {
+                let partitions = descriptor
+                    .partitions
+                    .iter()
+                    .map(|partition| {
+                        let row_count = usize::try_from(partition.row_count).map_err(|_| {
+                            datafusion_common::DataFusionError::Plan(
+                                "checkpoint row count is too large".to_string(),
+                            )
+                        })?;
+                        let batch = RecordBatch::try_new_with_options(
+                            Arc::clone(&descriptor.storage_schema),
+                            vec![],
+                            &RecordBatchOptions::new().with_row_count(Some(row_count)),
+                        )?;
+                        Ok(vec![batch])
+                    })
+                    .collect::>>()?;
+                let source = MemorySourceConfig::try_new(
+                    &partitions,
+                    Arc::clone(&descriptor.storage_schema),
+                    None,
+                )?
+                .try_with_sort_information(storage_output_ordering.clone().into_iter().collect())?;
+                Arc::new(DataSourceExec::new(Arc::new(CheckpointDataSource::new(
+                    Arc::new(source),
+                    output_partitioning,
+                ))))
+            } else {
+                let parquet_options = TableParquetOptions {
+                    global: session_state.config().options().execution.parquet.clone(),
+                    ..Default::default()
+                };
+                let source = ParquetSource::new(TableSchema::new(
+                    Arc::clone(&descriptor.storage_schema),
+                    vec![],
+                ))
+                .with_table_parquet_options(parquet_options);
+                let file_groups = descriptor
+                    .partitions
+                    .iter()
+                    .map(|partition| {
+                        let files = partition
+                            .file
+                            .as_ref()
+                            .map(|file| {
+                                let mut statistics =
+                                    Statistics::new_unknown(descriptor.storage_schema.as_ref());
+                                statistics.num_rows = Precision::Exact(
+                                    usize::try_from(partition.row_count).map_err(|_| {
+                                        datafusion_common::DataFusionError::Plan(
+                                            "checkpoint row count is too large".to_string(),
+                                        )
+                                    })?,
+                                );
+                                Ok(PartitionedFile::new(file.location.to_string(), file.size)
+                                    .with_statistics(Arc::new(statistics)))
+                            })
+                            .into_iter()
+                            .collect::>>()?;
+                        Ok(FileGroup::new(files))
+                    })
+                    .collect::>>()?;
+                let total_rows =
+                    descriptor
+                        .partitions
+                        .iter()
+                        .try_fold(0_u64, |total, partition| {
+                            total.checked_add(partition.row_count).ok_or_else(|| {
+                                datafusion_common::DataFusionError::Plan(
+                                    "checkpoint row count overflow".to_string(),
+                                )
+                            })
+                        })?;
+                let mut statistics = Statistics::new_unknown(descriptor.storage_schema.as_ref());
+                statistics.num_rows =
+                    Precision::Exact(usize::try_from(total_rows).map_err(|_| {
+                        datafusion_common::DataFusionError::Plan(
+                            "checkpoint row count is too large".to_string(),
+                        )
+                    })?);
+                let scan = FileScanConfigBuilder::new(
+                    descriptor.object_store_url.clone(),
+                    Arc::new(source),
+                )
+                .with_file_groups(file_groups)
+                .with_statistics(statistics)
+                .with_output_ordering(storage_output_ordering.into_iter().collect())
+                .with_preserve_order(true)
+                .with_partitioned_by_file_group(true)
+                .build();
+                let projection = descriptor
+                    .storage_schema
+                    .fields()
+                    .iter()
+                    .zip(logical_schema.fields())
+                    .enumerate()
+                    .map(|(index, (storage, logical))| {
+                        ProjectionExpr::new(
+                            Arc::new(Column::new(storage.name(), index)) as Arc,
+                            logical.name(),
+                        )
+                    })
+                    .collect::();
+                let source = scan
+                    .try_swapping_with_projection(&projection)?
+                    .ok_or_else(|| {
+                        internal_datafusion_err!(
+                            "checkpoint Parquet source rejected its column projection"
+                        )
+                    })?;
+                Arc::new(DataSourceExec::new(Arc::new(CheckpointDataSource::new(
+                    source,
+                    output_partitioning,
+                ))))
+            };
+            scan
+        } else if let Some(node) = node.as_any().downcast_ref::() {
             let schema = UserDefinedLogicalNode::schema(node).inner().clone();
             let projection = (0..schema.fields().len()).collect();
             Arc::new(RangeExec::try_new(
@@ -162,6 +371,29 @@ impl ExtensionPlanner for ExtensionPhysicalPlanner {
                 .with_fetch(node.fetch())
                 .with_preserve_partitioning(true);
             Arc::new(sort)
+        } else if let Some(node) = node.as_any().downcast_ref::() {
+            let [input] = physical_inputs else {
+                return internal_err!("RequiredSort requires exactly one physical input");
+            };
+            let expr = create_physical_sort_exprs(
+                node.sort_expr(),
+                UserDefinedLogicalNode::schema(node),
+                session_state.execution_props(),
+            )?;
+            let Some(ordering) = LexOrdering::new(expr) else {
+                return internal_err!("RequiredSort requires at least one sort expression");
+            };
+            let sort = SortExec::new(ordering, input.clone())
+                .with_fetch(node.fetch())
+                .with_preserve_partitioning(node.preserve_partitioning());
+            let requirements = OrderingRequirements::from(sort.expr().clone());
+            let distribution = sort.required_input_distribution().swap_remove(0);
+            Arc::new(OutputRequirementExec::new(
+                Arc::new(sort),
+                Some(requirements),
+                distribution,
+                node.fetch(),
+            ))
         } else if let Some(node) = node.as_any().downcast_ref::() {
             let [input] = physical_inputs else {
                 return internal_err!("SchemaPivotExec requires exactly one physical input");
@@ -259,6 +491,66 @@ impl ExtensionPlanner for ExtensionPhysicalPlanner {
     }
 }
 
+// Column ordinals survive logical/storage renames; names may be duplicated or synthetic.
+fn checkpoint_schema_partitioning(
+    partitioning: &Partitioning,
+    schema: &datafusion::arrow::datatypes::Schema,
+) -> datafusion_common::Result {
+    match partitioning {
+        Partitioning::RoundRobinBatch(partitions) => Ok(Partitioning::RoundRobinBatch(*partitions)),
+        Partitioning::Hash(expressions, partitions) => Ok(Partitioning::Hash(
+            expressions
+                .iter()
+                .map(|expression| checkpoint_schema_expression(Arc::clone(expression), schema))
+                .collect::>>()?,
+            *partitions,
+        )),
+        Partitioning::UnknownPartitioning(partitions) => {
+            Ok(Partitioning::UnknownPartitioning(*partitions))
+        }
+    }
+}
+
+fn checkpoint_schema_ordering(
+    ordering: &LexOrdering,
+    schema: &datafusion::arrow::datatypes::Schema,
+) -> datafusion_common::Result {
+    let expressions = ordering
+        .iter()
+        .map(|sort| {
+            Ok(PhysicalSortExpr::new(
+                checkpoint_schema_expression(Arc::clone(&sort.expr), schema)?,
+                sort.options,
+            ))
+        })
+        .collect::>>()?;
+    LexOrdering::new(expressions)
+        .ok_or_else(|| internal_datafusion_err!("checkpoint output ordering cannot be empty"))
+}
+
+fn checkpoint_schema_expression(
+    expression: Arc,
+    schema: &datafusion::arrow::datatypes::Schema,
+) -> datafusion_common::Result> {
+    expression
+        .transform_down(|expression| {
+            let Some(column) = expression.downcast_ref::() else {
+                return Ok(Transformed::no(expression));
+            };
+            let field = schema.fields().get(column.index()).ok_or_else(|| {
+                internal_datafusion_err!(
+                    "checkpoint property references column {} at invalid index {}",
+                    column.name(),
+                    column.index()
+                )
+            })?;
+            Ok(Transformed::yes(
+                Arc::new(Column::new(field.name(), column.index())) as Arc,
+            ))
+        })
+        .data()
+}
+
 /// Plans the explicit repartitioning emitted by the logical planner.
 /// Empty expressions keep round-robin explicit here, while hash repartitioning
 /// and `UnknownPartitioning(1)` are normalized later by `RewriteExplicitRepartition`.
@@ -318,11 +610,20 @@ mod tests {
     use std::sync::Arc;
 
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::execution::SessionStateBuilder;
+    use datafusion::execution::runtime_env::RuntimeEnv;
+    use datafusion::physical_expr::expressions::Column;
+    use datafusion::physical_expr::{PhysicalExpr, PhysicalSortExpr};
+    use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+    use datafusion::physical_plan::collect;
     use datafusion::physical_plan::empty::EmptyExec;
     use datafusion::physical_plan::repartition::RepartitionExec;
-    use datafusion::prelude::SessionContext;
+    use datafusion::prelude::{SessionConfig, SessionContext};
     use datafusion_common::ToDFSchema;
-    use datafusion_expr::col;
+    use datafusion_expr::{LogicalPlanBuilder, col};
+    use object_store::memory::InMemory;
+    use sail_common::spec;
+    use sail_plan::config::PlanConfig;
 
     use super::*;
 
@@ -330,6 +631,224 @@ mod tests {
         Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]))
     }
 
+    #[test]
+    fn checkpoint_properties_map_columns_to_target_schema_by_position() {
+        let target_schema = Schema::new(vec![
+            Field::new("_c0", DataType::Int64, false),
+            Field::new("_c1", DataType::Int64, false),
+        ]);
+        let first = Arc::new(Column::new("value", 0)) as Arc;
+        let second = Arc::new(Column::new("value", 1)) as Arc;
+        let ordering = LexOrdering::new([PhysicalSortExpr::new_default(first)]).unwrap();
+
+        let partitioning =
+            checkpoint_schema_partitioning(&Partitioning::Hash(vec![second], 2), &target_schema)
+                .unwrap();
+        let ordering = checkpoint_schema_ordering(&ordering, &target_schema).unwrap();
+
+        let Partitioning::Hash(expressions, 2) = partitioning else {
+            panic!("expected hash partitioning");
+        };
+        let partition_column = expressions[0].downcast_ref::().unwrap();
+        assert_eq!(partition_column.name(), "_c1");
+        assert_eq!(partition_column.index(), 1);
+        let ordering_column = ordering.first().expr.downcast_ref::().unwrap();
+        assert_eq!(ordering_column.name(), "_c0");
+        assert_eq!(ordering_column.index(), 0);
+    }
+
+    #[tokio::test]
+    async fn checkpoint_command_optimizer_coalesces_writer_before_commit()
+    -> datafusion_common::Result<()> {
+        let registry = Arc::new(RemoteCheckpointRegistry::new(
+            Some("memory:///checkpoint".to_string()),
+            "session".to_string(),
+        ));
+        let config = SessionConfig::new().with_extension(registry);
+        let runtime_env = Arc::new(RuntimeEnv::default());
+        let object_store_url =
+            datafusion::execution::object_store::ObjectStoreUrl::parse("memory://")?;
+        runtime_env.register_object_store(object_store_url.as_ref(), Arc::new(InMemory::new()));
+        let state = SessionStateBuilder::new()
+            .with_config(config)
+            .with_runtime_env(runtime_env)
+            .with_default_features()
+            .with_query_planner(new_query_planner())
+            .build();
+        let context = SessionContext::new_with_state(state);
+        let command = spec::Plan::Command(spec::CommandPlan::new(
+            spec::CommandNode::RemoteCheckpoint {
+                relation_id: "relation".to_string(),
+                input: Box::new(spec::QueryPlan::new(spec::QueryNode::Range(spec::Range {
+                    start: Some(0),
+                    end: 10,
+                    step: 1,
+                    num_partitions: Some(2),
+                }))),
+            },
+        ));
+
+        let (physical, _) =
+            sail_plan::resolve_and_execute_plan(&context, Arc::new(PlanConfig::default()), command)
+                .await
+                .map_err(|error| internal_datafusion_err!("{error}"))?;
+
+        let commit = physical
+            .downcast_ref::()
+            .ok_or_else(|| internal_datafusion_err!("checkpoint command did not plan a commit"))?;
+        let coalesce = commit
+            .input()
+            .downcast_ref::()
+            .ok_or_else(|| internal_datafusion_err!("checkpoint commit input is not coalesced"))?;
+        assert!(coalesce.input().is::());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn zero_column_checkpoint_scan_preserves_partition_rows() -> datafusion_common::Result<()>
+    {
+        let registry = Arc::new(RemoteCheckpointRegistry::new(
+            Some("memory:///checkpoint".to_string()),
+            "session".to_string(),
+        ));
+        registry.insert(sail_cache::remote_checkpoint::RemoteCheckpointDescriptor {
+            relation_id: "zero-columns".to_string(),
+            object_store_url: datafusion::execution::object_store::ObjectStoreUrl::parse(
+                "memory://",
+            )?,
+            prefix: object_store::path::Path::from("checkpoint/session/zero-columns"),
+            logical_schema: Arc::new(Schema::empty()),
+            storage_schema: Arc::new(Schema::empty()),
+            output_partitioning: Partitioning::RoundRobinBatch(2),
+            output_ordering: None,
+            partitions: vec![
+                sail_cache::remote_checkpoint::RemoteCheckpointPartition {
+                    partition: 0,
+                    row_count: 2,
+                    file: None,
+                },
+                sail_cache::remote_checkpoint::RemoteCheckpointPartition {
+                    partition: 1,
+                    row_count: 3,
+                    file: None,
+                },
+            ],
+        })?;
+        let config = SessionConfig::new().with_extension(registry);
+        let state = SessionStateBuilder::new()
+            .with_config(config)
+            .with_default_features()
+            .with_query_planner(new_query_planner())
+            .build();
+        let context = SessionContext::new_with_state(state);
+        let logical = LogicalPlan::Extension(datafusion_expr::Extension {
+            node: Arc::new(RemoteCheckpointRelationNode::try_new(
+                "zero-columns".to_string(),
+                Arc::new(Schema::empty()),
+            )?),
+        });
+        let state = context.state();
+        let physical = state
+            .query_planner()
+            .create_physical_plan(&logical, &state)
+            .await?;
+
+        assert!(physical.is::());
+        assert!(matches!(
+            physical.output_partitioning(),
+            Partitioning::RoundRobinBatch(2)
+        ));
+        let batches = collect(physical, context.task_ctx()).await?;
+        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 5);
+        assert!(batches.iter().all(|batch| batch.num_columns() == 0));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn checkpoint_scan_uses_data_source_properties() -> datafusion_common::Result<()> {
+        let logical_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+        let storage_schema = checkpoint_storage_schema(&logical_schema);
+        let storage_key = Arc::new(Column::new("_c0", 0)) as Arc;
+        let registry = Arc::new(RemoteCheckpointRegistry::new(
+            Some("memory:///checkpoint".to_string()),
+            "session".to_string(),
+        ));
+        registry.insert(sail_cache::remote_checkpoint::RemoteCheckpointDescriptor {
+            relation_id: "relation".to_string(),
+            object_store_url: datafusion::execution::object_store::ObjectStoreUrl::parse(
+                "memory://",
+            )?,
+            prefix: object_store::path::Path::from("checkpoint/session/relation"),
+            logical_schema: Arc::clone(&logical_schema),
+            storage_schema,
+            output_partitioning: Partitioning::Hash(vec![Arc::clone(&storage_key)], 2),
+            output_ordering: LexOrdering::new([PhysicalSortExpr::new_default(storage_key)]),
+            partitions: (0..2)
+                .map(
+                    |partition| sail_cache::remote_checkpoint::RemoteCheckpointPartition {
+                        partition,
+                        row_count: 0,
+                        file: None,
+                    },
+                )
+                .collect(),
+        })?;
+        let config = SessionConfig::new().with_extension(registry);
+        let state = SessionStateBuilder::new()
+            .with_config(config)
+            .with_default_features()
+            .with_query_planner(new_query_planner())
+            .build();
+        let context = SessionContext::new_with_state(state);
+        let logical = LogicalPlan::Extension(datafusion_expr::Extension {
+            node: Arc::new(RemoteCheckpointRelationNode::try_new(
+                "relation".to_string(),
+                logical_schema,
+            )?),
+        });
+        let state = context.state();
+        let physical = state
+            .query_planner()
+            .create_physical_plan(&logical, &state)
+            .await?;
+
+        assert!(physical.is::());
+        let Partitioning::Hash(expressions, 2) = physical.output_partitioning() else {
+            return internal_err!("checkpoint scan lost its hash partitioning");
+        };
+        let partition_column = expressions[0]
+            .downcast_ref::()
+            .ok_or_else(|| internal_datafusion_err!("checkpoint hash key is not a column"))?;
+        assert_eq!(partition_column.name(), "id");
+        assert_eq!(partition_column.index(), 0);
+        let ordering = physical
+            .output_ordering()
+            .ok_or_else(|| internal_datafusion_err!("checkpoint scan lost its ordering"))?;
+        let ordering_column = ordering[0]
+            .expr
+            .downcast_ref::()
+            .ok_or_else(|| internal_datafusion_err!("checkpoint ordering key is not a column"))?;
+        assert_eq!(ordering_column.name(), "id");
+        assert_eq!(ordering_column.index(), 0);
+
+        let projected_logical = LogicalPlanBuilder::from(logical)
+            .project(vec![col("id").alias("key")])?
+            .build()?;
+        let projected = state
+            .query_planner()
+            .create_physical_plan(&projected_logical, &state)
+            .await?;
+        let Partitioning::Hash(expressions, 2) = projected.output_partitioning() else {
+            return internal_err!("checkpoint projection lost its hash partitioning");
+        };
+        let partition_column = expressions[0]
+            .downcast_ref::()
+            .ok_or_else(|| internal_datafusion_err!("projected hash key is not a column"))?;
+        assert_eq!(partition_column.name(), "key");
+        assert_eq!(partition_column.index(), 0);
+        Ok(())
+    }
+
     fn plan_partitioning(
         input: &dyn ExecutionPlan,
         schema: &Arc,
@@ -400,9 +919,11 @@ mod tests {
         )
         .unwrap_err();
 
-        assert!(error
-            .to_string()
-            .contains("number of explicit partitions cannot be zero"));
+        assert!(
+            error
+                .to_string()
+                .contains("number of explicit partitions cannot be zero")
+        );
     }
 
     #[test]
@@ -419,9 +940,11 @@ mod tests {
         )
         .unwrap_err();
 
-        assert!(error
-            .to_string()
-            .contains("explicit round-robin repartition requires a target partition count"));
+        assert!(
+            error
+                .to_string()
+                .contains("explicit round-robin repartition requires a target partition count")
+        );
     }
 
     #[test]
diff --git a/crates/sail-session/src/runtime.rs b/crates/sail-session/src/runtime.rs
index 3323cd5f6..5616bf7a6 100644
--- a/crates/sail-session/src/runtime.rs
+++ b/crates/sail-session/src/runtime.rs
@@ -1,5 +1,6 @@
 use std::sync::Arc;
 
+use datafusion::execution::DiskManager;
 use datafusion::execution::cache::cache_manager::{
     CacheManagerConfig, FileMetadataCache, FileStatisticsCache, ListFilesCache,
 };
@@ -8,7 +9,6 @@ use datafusion::execution::memory_pool::{
     FairSpillPool, GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
 };
 use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
-use datafusion::execution::DiskManager;
 use datafusion_common::Result;
 use log::debug;
 use sail_cache::file_listing_cache::MokaFileListingCache;
diff --git a/crates/sail-session/src/session_factory/job_runner.rs b/crates/sail-session/src/session_factory/job_runner.rs
new file mode 100644
index 000000000..49c72bd31
--- /dev/null
+++ b/crates/sail-session/src/session_factory/job_runner.rs
@@ -0,0 +1,139 @@
+use std::sync::Arc;
+
+use datafusion::common::{Result, internal_err};
+use sail_common::actor::ActorSystem;
+use sail_common::config::{AppConfig, ExecutionMode};
+use sail_common::runtime::RuntimeHandle;
+use sail_common_datafusion::session::job::{JobRunner, JobRunnerHistoryReporter};
+use sail_execution::DriverId;
+use sail_execution::driver::{DriverComponents, DriverHandle, DriverOptions};
+use sail_execution::job_runner::{ClusterJobRunner, LocalJobRunner};
+use sail_execution::worker_manager::{
+    KubernetesWorkerManager, KubernetesWorkerManagerOptions, LocalWorkerManager,
+};
+
+use crate::session_factory::{SessionFactory, WorkerSessionFactory};
+
+pub struct SessionJobRunner {
+    runner: Box,
+    driver: Option,
+}
+
+impl SessionJobRunner {
+    fn local(runner: LocalJobRunner) -> Self {
+        Self {
+            runner: Box::new(runner),
+            driver: None,
+        }
+    }
+
+    fn cluster(runner: ClusterJobRunner) -> Self {
+        let driver = runner.driver();
+        Self {
+            runner: Box::new(runner),
+            driver: Some(driver),
+        }
+    }
+
+    pub(crate) fn into_parts(self) -> (Box, Option) {
+        (self.runner, self.driver)
+    }
+}
+
+pub struct SessionJobRunnerInfo {
+    pub session_id: String,
+    pub driver_id: DriverId,
+    pub driver_server_port: Option,
+    pub history_reporter: Box,
+}
+
+pub trait SessionJobRunnerFactory: Send {
+    fn create(
+        &mut self,
+        system: &mut ActorSystem,
+        info: SessionJobRunnerInfo,
+    ) -> Result;
+}
+
+pub struct ServerSessionJobRunnerFactory {
+    config: Arc,
+    runtime: RuntimeHandle,
+}
+
+impl ServerSessionJobRunnerFactory {
+    pub fn new(config: Arc, runtime: RuntimeHandle) -> Self {
+        Self { config, runtime }
+    }
+
+    fn create_cluster_runner(
+        &self,
+        system: &mut ActorSystem,
+        info: SessionJobRunnerInfo,
+        worker_manager: Box,
+    ) -> Result {
+        let Some(port) = info.driver_server_port else {
+            return internal_err!("driver gateway is not available");
+        };
+        let options = DriverOptions::new(
+            &self.config,
+            self.runtime.clone(),
+            info.session_id,
+            info.driver_id,
+            port,
+        );
+        let components = DriverComponents {
+            worker_manager,
+            history_reporter: info.history_reporter,
+        };
+        Ok(SessionJobRunner::cluster(ClusterJobRunner::new(
+            system, options, components,
+        )))
+    }
+}
+
+impl SessionJobRunnerFactory for ServerSessionJobRunnerFactory {
+    fn create(
+        &mut self,
+        system: &mut ActorSystem,
+        info: SessionJobRunnerInfo,
+    ) -> Result {
+        match self.config.mode {
+            ExecutionMode::Local => Ok(SessionJobRunner::local(LocalJobRunner::new(
+                info.history_reporter,
+            ))),
+            ExecutionMode::LocalCluster => {
+                let worker_session =
+                    WorkerSessionFactory::new(self.config.clone(), self.runtime.clone())
+                        .create(())?;
+                self.create_cluster_runner(
+                    system,
+                    info,
+                    Box::new(LocalWorkerManager::new(
+                        self.runtime.clone(),
+                        worker_session,
+                    )),
+                )
+            }
+            ExecutionMode::KubernetesCluster => {
+                let options = KubernetesWorkerManagerOptions {
+                    image: self.config.kubernetes.image.clone(),
+                    image_pull_policy: self.config.kubernetes.image_pull_policy.clone(),
+                    namespace: self.config.kubernetes.namespace.clone(),
+                    driver_pod_name: self.config.kubernetes.driver_pod_name.clone(),
+                    worker_pod_name_prefix: self.config.kubernetes.worker_pod_name_prefix.clone(),
+                    worker_service_account_name: self
+                        .config
+                        .kubernetes
+                        .worker_service_account_name
+                        .clone(),
+                    worker_pod_template: self.config.kubernetes.worker_pod_template.clone(),
+                };
+                self.create_cluster_runner(
+                    system,
+                    info,
+                    Box::new(KubernetesWorkerManager::new(options)),
+                )
+            }
+        }
+    }
+}
diff --git a/crates/sail-session/src/session_factory/mod.rs b/crates/sail-session/src/session_factory/mod.rs
index 73a8c7e7f..504c34fb6 100644
--- a/crates/sail-session/src/session_factory/mod.rs
+++ b/crates/sail-session/src/session_factory/mod.rs
@@ -1,8 +1,12 @@
+mod job_runner;
 mod server;
 mod worker;
 
 use datafusion::common::Result;
 use datafusion::prelude::SessionContext;
+pub use job_runner::{
+    ServerSessionJobRunnerFactory, SessionJobRunner, SessionJobRunnerFactory, SessionJobRunnerInfo,
+};
 pub use server::{ServerSessionFactory, ServerSessionInfo, ServerSessionMutator};
 pub use worker::WorkerSessionFactory;
 
diff --git a/crates/sail-session/src/session_factory/server.rs b/crates/sail-session/src/session_factory/server.rs
index 4d7584d74..68509ad30 100644
--- a/crates/sail-session/src/session_factory/server.rs
+++ b/crates/sail-session/src/session_factory/server.rs
@@ -1,29 +1,24 @@
-use std::ops::DerefMut;
 use std::str::FromStr;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 
 use datafusion::common::parquet_config::DFParquetWriterVersion;
-use datafusion::common::{internal_datafusion_err, Result};
+use datafusion::common::{Result, internal_err};
 use datafusion::execution::runtime_env::RuntimeEnvBuilder;
 use datafusion::execution::{SessionState, SessionStateBuilder};
 use datafusion::functions_aggregate::first_last::first_value_udaf;
 use datafusion::prelude::{SessionConfig, SessionContext};
 use datafusion_expr::registry::FunctionRegistry;
+use sail_cache::remote_checkpoint::RemoteCheckpointRegistry;
 use sail_catalog::provider::CatalogCacheManager;
 use sail_catalog_system::service::SystemTableService;
-use sail_common::config::{AppConfig, ExecutionMode};
+use sail_common::actor::ActorHandle;
+use sail_common::config::AppConfig;
 use sail_common::runtime::RuntimeHandle;
 use sail_common_datafusion::session::activity::ActivityTracker;
 use sail_common_datafusion::session::job::{JobRunner, JobService};
 use sail_common_datafusion::session::repartition::RepartitionBufferConfig;
 use sail_delta_lake::session_extension::DeltaTableCache;
-use sail_execution::driver::DriverOptions;
-use sail_execution::job_runner::{ClusterJobRunner, LocalJobRunner};
-use sail_execution::worker_manager::{
-    KubernetesWorkerManager, KubernetesWorkerManagerOptions, LocalWorkerManager,
-};
-use sail_physical_optimizer::{get_physical_optimizers, PhysicalOptimizerOptions};
-use sail_server::actor::{ActorHandle, ActorSystem};
+use sail_physical_optimizer::{PhysicalOptimizerOptions, get_physical_optimizers};
 
 use crate::catalog::create_catalog_manager;
 use crate::formats::create_table_format_registry;
@@ -31,13 +26,14 @@ use crate::observable::SessionManagerHandle;
 use crate::optimizer::{default_analyzer_rules, default_optimizer_rules};
 use crate::planner::new_query_planner;
 use crate::runtime::RuntimeEnvFactory;
-use crate::session_factory::{SessionFactory, WorkerSessionFactory};
+use crate::session_factory::SessionFactory;
 use crate::session_manager::SessionManagerActor;
 
 pub struct ServerSessionInfo {
     pub session_id: String,
     pub user_id: String,
     pub session_manager: ActorHandle,
+    pub job_runner: Option>,
 }
 
 pub trait ServerSessionMutator: Send {
@@ -61,7 +57,6 @@ pub trait ServerSessionMutator: Send {
 pub struct ServerSessionFactory {
     config: Arc,
     runtime: RuntimeHandle,
-    system: Arc>,
     mutator: Box,
     runtime_env: RuntimeEnvFactory,
     catalog_cache_manager: Arc,
@@ -71,14 +66,12 @@ impl ServerSessionFactory {
     pub fn new(
         config: Arc,
         runtime: RuntimeHandle,
-        system: Arc>,
         mutator: Box,
     ) -> Self {
         let runtime_env = RuntimeEnvFactory::new(config.clone(), runtime.clone());
         Self {
             config,
             runtime,
-            system,
             mutator,
             runtime_env,
             catalog_cache_manager: Arc::new(CatalogCacheManager::new()),
@@ -87,8 +80,8 @@ impl ServerSessionFactory {
 }
 
 impl SessionFactory for ServerSessionFactory {
-    fn create(&mut self, info: ServerSessionInfo) -> Result {
-        let state = self.create_session_state(&info)?;
+    fn create(&mut self, mut info: ServerSessionInfo) -> Result {
+        let state = self.create_session_state(&mut info)?;
         let context = SessionContext::new_with_state(state);
 
         // Register the `first_value` UDAF since the `replace_distinct_aggregate` optimizer rule
@@ -108,8 +101,10 @@ impl SessionFactory for ServerSessionFactory {
 }
 
 impl ServerSessionFactory {
-    fn create_session_config(&mut self, info: &ServerSessionInfo) -> Result {
-        let job_runner = self.create_job_runner()?;
+    fn create_session_config(&mut self, info: &mut ServerSessionInfo) -> Result {
+        let Some(job_runner) = info.job_runner.take() else {
+            return internal_err!("job runner is missing from server session information");
+        };
         let mut config = SessionConfig::new()
             // We do not use the DataFusion catalog and schema since we manage catalogs ourselves.
             .with_create_default_catalog_and_schema(false)
@@ -122,6 +117,10 @@ impl ServerSessionFactory {
             )?))
             .with_extension(Arc::new(ActivityTracker::new()))
             .with_extension(Arc::new(JobService::new(job_runner)))
+            .with_extension(Arc::new(RemoteCheckpointRegistry::new(
+                self.config.execution.checkpoint.path.clone(),
+                info.session_id.clone(),
+            )))
             .with_extension(Arc::new(RepartitionBufferConfig::new(
                 self.config.cluster.task_stream_buffer,
             )))
@@ -134,7 +133,7 @@ impl ServerSessionFactory {
         Ok(config)
     }
 
-    fn create_session_state(&mut self, info: &ServerSessionInfo) -> Result {
+    fn create_session_state(&mut self, info: &mut ServerSessionInfo) -> Result {
         let config = self.create_session_config(info)?;
         let runtime = self
             .runtime_env
@@ -155,50 +154,6 @@ impl ServerSessionFactory {
         Ok(builder.build())
     }
 
-    fn create_job_runner(&mut self) -> Result> {
-        let job_runner: Box = match self.config.mode {
-            ExecutionMode::Local => Box::new(LocalJobRunner::new()),
-            ExecutionMode::LocalCluster => {
-                let worker_manager = Arc::new(LocalWorkerManager::new(
-                    self.runtime.clone(),
-                    WorkerSessionFactory::new(self.config.clone(), self.runtime.clone())
-                        .create(())?,
-                ));
-                let options =
-                    DriverOptions::new(&self.config, self.runtime.clone(), worker_manager);
-                let mut system = self
-                    .system
-                    .lock()
-                    .map_err(|e| internal_datafusion_err!("{e}"))?;
-                Box::new(ClusterJobRunner::new(system.deref_mut(), options))
-            }
-            ExecutionMode::KubernetesCluster => {
-                let options = KubernetesWorkerManagerOptions {
-                    image: self.config.kubernetes.image.clone(),
-                    image_pull_policy: self.config.kubernetes.image_pull_policy.clone(),
-                    namespace: self.config.kubernetes.namespace.clone(),
-                    driver_pod_name: self.config.kubernetes.driver_pod_name.clone(),
-                    worker_pod_name_prefix: self.config.kubernetes.worker_pod_name_prefix.clone(),
-                    worker_service_account_name: self
-                        .config
-                        .kubernetes
-                        .worker_service_account_name
-                        .clone(),
-                    worker_pod_template: self.config.kubernetes.worker_pod_template.clone(),
-                };
-                let worker_manager = Arc::new(KubernetesWorkerManager::new(options));
-                let options =
-                    DriverOptions::new(&self.config, self.runtime.clone(), worker_manager);
-                let mut system = self
-                    .system
-                    .lock()
-                    .map_err(|e| internal_datafusion_err!("{e}"))?;
-                Box::new(ClusterJobRunner::new(system.deref_mut(), options))
-            }
-        };
-        Ok(job_runner)
-    }
-
     fn create_system_table_service(&self, info: &ServerSessionInfo) -> Result {
         Ok(SystemTableService::new(Box::new(
             SessionManagerHandle::new(info.session_manager.clone()),
diff --git a/crates/sail-session/src/session_manager/actor/core.rs b/crates/sail-session/src/session_manager/actor/core.rs
index c06ec4458..f06ee8b52 100644
--- a/crates/sail-session/src/session_manager/actor/core.rs
+++ b/crates/sail-session/src/session_manager/actor/core.rs
@@ -1,28 +1,73 @@
+use std::sync::Arc;
+
 use indexmap::IndexMap;
-use sail_server::actor::{Actor, ActorAction, ActorContext};
+use log::{info, warn};
+use sail_common::actor::{Actor, ActorAction, ActorContext, ActorHandle};
+use sail_execution::driver::{DriverHandle, DriverRegistryAccessor};
+use sail_execution::error::{ExecutionError, ExecutionResult};
+use sail_execution::{DriverId, IdGenerator};
 
 use crate::session_manager::actor::SessionManagerActor;
 use crate::session_manager::event::SessionManagerEvent;
-use crate::session_manager::options::SessionManagerOptions;
+use crate::session_manager::options::{SessionManagerComponents, SessionManagerOptions};
+
+struct SessionDriverRegistry {
+    handle: ActorHandle,
+}
+
+#[tonic::async_trait]
+impl DriverRegistryAccessor for SessionDriverRegistry {
+    async fn get(&self, driver_id: DriverId) -> ExecutionResult {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+        self.handle
+            .send(SessionManagerEvent::GetDriver {
+                driver_id,
+                result: tx,
+            })
+            .await
+            .map_err(ExecutionError::from)?;
+        rx.await.map_err(ExecutionError::from)?
+    }
+}
 
 #[tonic::async_trait]
 impl Actor for SessionManagerActor {
     type Message = SessionManagerEvent;
-    type Options = SessionManagerOptions;
+    type Options = (SessionManagerOptions, SessionManagerComponents);
 
     fn name() -> &'static str {
         "SessionManagerActor"
     }
 
     fn new(options: Self::Options) -> Self {
-        let factory = (options.factory)();
+        let (options, components) = options;
+        let SessionManagerComponents {
+            session_factory,
+            job_runner_factory,
+            driver_gateway,
+        } = components;
         Self {
             options,
-            factory,
+            session_factory,
+            job_runner_factory,
             sessions: IndexMap::new(),
+            drivers: Default::default(),
+            driver_gateway,
+            driver_id_generator: IdGenerator::new(),
+            shutdown_notifier: None,
         }
     }
 
+    async fn start(&mut self, ctx: &mut ActorContext) {
+        let Some(driver_gateway) = &mut self.driver_gateway else {
+            return;
+        };
+        driver_gateway.start(Arc::new(SessionDriverRegistry {
+            handle: ctx.handle().clone(),
+        }));
+        info!("driver server is ready on port {}", driver_gateway.port());
+    }
+
     fn receive(&mut self, ctx: &mut ActorContext, message: Self::Message) -> ActorAction {
         match message {
             SessionManagerEvent::GetOrCreateSession {
@@ -47,6 +92,32 @@ impl Actor for SessionManagerActor {
             SessionManagerEvent::ObserveState { observer } => {
                 self.handle_observe_state(ctx, observer)
             }
+            SessionManagerEvent::GetDriver { driver_id, result } => {
+                self.handle_get_driver(driver_id, result)
+            }
+            SessionManagerEvent::Shutdown { result } => {
+                self.shutdown_notifier = Some(result);
+                ActorAction::Stop
+            }
+        }
+    }
+
+    async fn stop(mut self, ctx: &mut ActorContext) {
+        // Keep the gateway available while drivers stop. Graceful gateway shutdown waits for
+        // active task stream connections, which are owned by the drivers.
+        let drivers = self.drivers.drain().collect::>();
+        for (driver_id, driver) in drivers {
+            if let Err(e) = driver.shutdown_and_wait().await {
+                warn!("failed to shut down driver {driver_id}: {e}");
+            }
+        }
+        ctx.children_mut().join().await;
+        if let Some(mut driver_gateway) = self.driver_gateway {
+            driver_gateway.stop().await;
+            info!("driver server has stopped");
+        }
+        if let Some(result) = self.shutdown_notifier {
+            let _ = result.send(());
         }
     }
 }
diff --git a/crates/sail-session/src/session_manager/actor/handler.rs b/crates/sail-session/src/session_manager/actor/handler.rs
index 900bff95d..a9c684026 100644
--- a/crates/sail-session/src/session_manager/actor/handler.rs
+++ b/crates/sail-session/src/session_manager/actor/handler.rs
@@ -2,26 +2,61 @@ use std::sync::Arc;
 
 use chrono::Utc;
 use datafusion::prelude::SessionContext;
-use fastrace::collector::SpanContext;
 use fastrace::Span;
+use fastrace::collector::SpanContext;
 use log::{info, warn};
+use sail_cache::remote_checkpoint::RemoteCheckpointRegistry;
+use sail_common::actor::{ActorAction, ActorContext};
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::activity::ActivityTracker;
-use sail_common_datafusion::session::job::JobService;
+use sail_common_datafusion::session::job::{
+    JobRunnerHistory, JobRunnerHistoryReporter, JobService,
+};
 use sail_common_datafusion::system::catalog::{OptionRow, SessionRow};
 use sail_common_datafusion::system::observable::{JobRunnerObserver, SessionManagerObserver};
 use sail_common_datafusion::system::predicate::PredicateExt;
-use sail_server::actor::{ActorAction, ActorContext};
+use sail_execution::DriverId;
+use sail_execution::driver::DriverHandle;
+use sail_execution::error::ExecutionResult;
 use tokio::sync::oneshot;
 use tokio::time::Instant;
 
 use crate::error::{SessionError, SessionResult};
-use crate::session_factory::ServerSessionInfo;
+use crate::session_factory::{ServerSessionInfo, SessionJobRunnerInfo};
 use crate::session_manager::actor::SessionManagerActor;
 use crate::session_manager::event::{SessionHistory, SessionManagerEvent};
 use crate::session_manager::session::{ServerSession, ServerSessionState};
 
+struct SessionJobRunnerHistoryReporter {
+    session_id: String,
+    session_manager: sail_common::actor::ActorHandle,
+}
+
+#[tonic::async_trait]
+impl JobRunnerHistoryReporter for SessionJobRunnerHistoryReporter {
+    async fn report(self: Box, history: JobRunnerHistory) {
+        let _ = self
+            .session_manager
+            .send(SessionManagerEvent::SetSessionHistory {
+                session_id: self.session_id,
+                history: SessionHistory {
+                    job_runner: history,
+                },
+            })
+            .await;
+    }
+}
+
 impl SessionManagerActor {
+    pub(super) fn handle_get_driver(
+        &self,
+        driver_id: DriverId,
+        result: oneshot::Sender>,
+    ) -> ActorAction {
+        let _ = result.send(self.drivers.get(driver_id));
+        ActorAction::Continue
+    }
+
     pub(super) fn handle_get_or_create_session(
         &mut self,
         ctx: &mut ActorContext,
@@ -30,7 +65,7 @@ impl SessionManagerActor {
         result: oneshot::Sender>,
     ) -> ActorAction {
         let context = if let Some(session) = self.sessions.get(&session_id) {
-            if let ServerSessionState::Running { context } = &session.state {
+            if let ServerSessionState::Running { context, .. } = &session.state {
                 Ok(context.clone())
             } else {
                 Err(SessionError::invalid(format!(
@@ -38,6 +73,10 @@ impl SessionManagerActor {
                 )))
             }
         } else {
+            // TODO: The session ID is used in various storage paths, so it is assumed to be unique
+            //   across all session managers, and it should contain only valid characters for a
+            //   path segment. Right now the session ID is generated as a UUID by the Spark client,
+            //   so this is true in practice, but we may still want some validation here.
             let session_id = session_id.clone();
             info!("creating session {session_id}");
             let span = Span::root(
@@ -45,40 +84,114 @@ impl SessionManagerActor {
                 SpanContext::random(),
             );
             let _guard = span.set_local_parent();
-            let info = ServerSessionInfo {
-                session_id: session_id.clone(),
-                user_id: user_id.clone(),
-                session_manager: ctx.handle().clone(),
+            let driver_id = match self.driver_id_generator.generate() {
+                Ok(driver_id) => driver_id,
+                Err(e) => {
+                    let output = Err(SessionError::internal(e.to_string()));
+                    let _ = result.send(output);
+                    return ActorAction::Continue;
+                }
             };
-            match self.factory.create(info) {
-                Ok(context) => {
-                    let session = ServerSession {
-                        user_id,
-                        created_at: Utc::now(),
-                        deleted_at: None,
-                        state: ServerSessionState::Running {
-                            context: context.clone(),
-                        },
+            let session_manager = ctx.handle().clone();
+            let runner = self.job_runner_factory.create(
+                ctx.children_mut(),
+                SessionJobRunnerInfo {
+                    session_id: session_id.clone(),
+                    driver_id,
+                    driver_server_port: self.driver_gateway.as_ref().map(|x| x.port()),
+                    history_reporter: Box::new(SessionJobRunnerHistoryReporter {
+                        session_id: session_id.clone(),
+                        session_manager,
+                    }),
+                },
+            );
+            match runner {
+                Ok(runner) => {
+                    let (runner, driver) = runner.into_parts();
+                    let registered_driver_id = driver.as_ref().map(|_| driver_id);
+                    if let Some(driver) = &driver
+                        && let Err(e) = self.drivers.insert(driver_id, driver.clone())
+                    {
+                        let session = ServerSession {
+                            user_id,
+                            created_at: Utc::now(),
+                            deleted_at: None,
+                            state: ServerSessionState::Failed,
+                        };
+                        self.sessions.insert(session_id, session);
+                        let driver = driver.clone();
+                        ctx.spawn(async move {
+                            if let Err(e) = driver.shutdown().await {
+                                warn!("failed to shut down driver {driver_id}: {e}");
+                            }
+                        });
+                        let output = Err(e.into());
+                        let _ = result.send(output);
+                        return ActorAction::Continue;
+                    }
+                    let info = ServerSessionInfo {
+                        session_id: session_id.clone(),
+                        user_id: user_id.clone(),
+                        session_manager: ctx.handle().clone(),
+                        job_runner: Some(runner),
                     };
-                    self.sessions.insert(session_id, session);
-                    Ok(context)
+                    match self.session_factory.create(info) {
+                        Ok(context) => {
+                            if let Some(driver) = driver {
+                                ctx.spawn(async move {
+                                    if let Err(e) = driver.activate().await {
+                                        warn!("failed to activate driver {driver_id}: {e}");
+                                    }
+                                });
+                            }
+                            let session = ServerSession {
+                                user_id,
+                                created_at: Utc::now(),
+                                deleted_at: None,
+                                state: ServerSessionState::Running {
+                                    context: context.clone(),
+                                    driver_id: registered_driver_id,
+                                },
+                            };
+                            self.sessions.insert(session_id, session);
+                            Ok(context)
+                        }
+                        Err(e) => {
+                            if let Some(driver_id) = registered_driver_id
+                                && let Some(driver) = self.drivers.remove(driver_id)
+                            {
+                                ctx.spawn(async move {
+                                    if let Err(e) = driver.shutdown().await {
+                                        warn!("failed to shut down driver {driver_id}: {e}");
+                                    }
+                                });
+                            }
+                            let session = ServerSession {
+                                user_id,
+                                created_at: Utc::now(),
+                                deleted_at: None,
+                                state: ServerSessionState::Failed,
+                            };
+                            self.sessions.insert(session_id, session);
+                            Err(e.into())
+                        }
+                    }
                 }
                 Err(e) => Err(e.into()),
             }
         };
-        if let Ok(context) = &context {
-            if let Ok(active_at) = context
+        if let Ok(context) = &context
+            && let Ok(active_at) = context
                 .extension::()
                 .and_then(|tracker| tracker.track_activity())
-            {
-                ctx.send_with_delay(
-                    SessionManagerEvent::ProbeIdleSession {
-                        session_id,
-                        instant: active_at,
-                    },
-                    self.options.session_timeout,
-                );
-            }
+        {
+            ctx.send_with_delay(
+                SessionManagerEvent::ProbeIdleSession {
+                    session_id,
+                    instant: active_at,
+                },
+                self.options.session_timeout,
+            );
         }
         let _ = result.send(context);
         ActorAction::Continue
@@ -91,17 +204,17 @@ impl SessionManagerActor {
         instant: Instant,
     ) -> ActorAction {
         let session = self.sessions.get_mut(&session_id);
-        if let Some(session) = session {
-            if let ServerSessionState::Running { context } = &mut session.state {
-                if let Ok(tracker) = context.extension::() {
-                    if tracker.active_at().is_ok_and(|x| x <= instant) {
-                        info!("removing idle session {session_id}");
-                        Self::delete_session(ctx, session_id, context);
-                        session.deleted_at = Some(Utc::now());
-                        session.state = ServerSessionState::Deleting;
-                    }
-                }
-            }
+        if let Some(session) = session
+            && let ServerSessionState::Running { context, driver_id } = &mut session.state
+            && let Ok(tracker) = context.extension::()
+            && tracker.active_at().is_ok_and(|x| x <= instant)
+        {
+            info!("removing idle session {session_id}");
+            Self::delete_session(ctx, session_id, context);
+            session.deleted_at = Some(Utc::now());
+            session.state = ServerSessionState::Deleting {
+                driver_id: *driver_id,
+            };
         }
         ActorAction::Continue
     }
@@ -114,11 +227,13 @@ impl SessionManagerActor {
     ) -> ActorAction {
         let session = self.sessions.get_mut(&session_id);
         let output = if let Some(session) = session {
-            if let ServerSessionState::Running { context } = &mut session.state {
+            if let ServerSessionState::Running { context, driver_id } = &mut session.state {
                 info!("removing session {session_id}");
                 Self::delete_session(ctx, session_id, context);
                 session.deleted_at = Some(Utc::now());
-                session.state = ServerSessionState::Deleting;
+                session.state = ServerSessionState::Deleting {
+                    driver_id: *driver_id,
+                };
                 Ok(())
             } else {
                 Err(SessionError::invalid(format!(
@@ -144,25 +259,47 @@ impl SessionManagerActor {
             warn!("session not found: {session_id}");
             return ActorAction::Continue;
         };
-        if matches!(session.state, ServerSessionState::Deleting) {
-            session.state = ServerSessionState::Deleted {
-                history: Arc::new(history),
-            };
-        } else {
-            warn!("session is not being deleted: {session_id}");
+        match &mut session.state {
+            ServerSessionState::Running { driver_id, .. }
+            | ServerSessionState::Deleting { driver_id } => {
+                if let Some(driver_id) = driver_id.take() {
+                    self.drivers.remove(driver_id);
+                }
+                session.deleted_at.get_or_insert_with(Utc::now);
+                session.state = ServerSessionState::Deleted {
+                    history: Arc::new(history),
+                };
+            }
+            ServerSessionState::Deleted { .. } | ServerSessionState::Failed => {
+                warn!("session is not being deleted: {session_id}");
+            }
         }
         ActorAction::Continue
     }
 
     pub(super) fn handle_set_session_failure(
         &mut self,
-        _ctx: &mut ActorContext,
+        ctx: &mut ActorContext,
         session_id: String,
     ) -> ActorAction {
         let Some(session) = self.sessions.get_mut(&session_id) else {
             warn!("session not found: {session_id}");
             return ActorAction::Continue;
         };
+        let driver_id = match &session.state {
+            ServerSessionState::Running { driver_id, .. }
+            | ServerSessionState::Deleting { driver_id } => *driver_id,
+            ServerSessionState::Deleted { .. } | ServerSessionState::Failed => None,
+        };
+        if let Some(driver_id) = driver_id
+            && let Some(driver) = self.drivers.remove(driver_id)
+        {
+            ctx.spawn(async move {
+                if let Err(e) = driver.shutdown().await {
+                    warn!("failed to shut down driver {driver_id}: {e}");
+                }
+            });
+        }
         session.state = ServerSessionState::Failed;
         ActorAction::Continue
     }
@@ -326,18 +463,18 @@ impl SessionManagerActor {
             warn!("job service not found for session {session_id}");
             return;
         };
-        let handle = ctx.handle().clone();
-        let (tx, rx) = oneshot::channel();
+        let checkpoint_registry = context.extension::().ok();
+        let runtime_env = context.runtime_env();
         ctx.spawn(async move {
-            service.runner().stop(tx).await;
-            let message = match rx.await {
-                Ok(x) => SessionManagerEvent::SetSessionHistory {
-                    session_id,
-                    history: SessionHistory { job_runner: x },
-                },
-                Err(_) => SessionManagerEvent::SetSessionFailure { session_id },
-            };
-            let _ = handle.send(message).await;
+            // Stop tasks before deleting the namespace so late attempts cannot recreate objects.
+            service.runner().stop().await;
+            if let Some(checkpoint_registry) = checkpoint_registry
+                && let Err(error) = checkpoint_registry
+                    .cleanup_session(runtime_env.as_ref())
+                    .await
+            {
+                warn!("failed to clean checkpoints for session {session_id}: {error}");
+            }
         });
     }
 }
diff --git a/crates/sail-session/src/session_manager/actor/mod.rs b/crates/sail-session/src/session_manager/actor/mod.rs
index 334bbc7de..b82d12744 100644
--- a/crates/sail-session/src/session_manager/actor/mod.rs
+++ b/crates/sail-session/src/session_manager/actor/mod.rs
@@ -2,12 +2,19 @@ mod core;
 mod handler;
 
 use indexmap::IndexMap;
+use sail_execution::driver::{DriverGateway, DriverRegistry};
+use sail_execution::{DriverId, IdGenerator};
 
-use crate::session_factory::{ServerSessionInfo, SessionFactory};
+use crate::session_factory::{ServerSessionInfo, SessionFactory, SessionJobRunnerFactory};
 use crate::session_manager::session::ServerSession;
 
 pub struct SessionManagerActor {
     options: super::options::SessionManagerOptions,
-    factory: Box>,
+    session_factory: Box>,
+    job_runner_factory: Box,
     sessions: IndexMap,
+    drivers: DriverRegistry,
+    driver_gateway: Option,
+    driver_id_generator: IdGenerator,
+    shutdown_notifier: Option>,
 }
diff --git a/crates/sail-session/src/session_manager/event.rs b/crates/sail-session/src/session_manager/event.rs
index 6f3c752d6..787db8629 100644
--- a/crates/sail-session/src/session_manager/event.rs
+++ b/crates/sail-session/src/session_manager/event.rs
@@ -1,9 +1,12 @@
 use std::borrow::Cow;
 
 use datafusion::prelude::SessionContext;
+use sail_common::telemetry::{SpanAssociation, SpanAttribute};
 use sail_common_datafusion::session::job::JobRunnerHistory;
 use sail_common_datafusion::system::observable::SessionManagerObserver;
-use sail_telemetry::common::{SpanAssociation, SpanAttribute};
+use sail_execution::DriverId;
+use sail_execution::driver::DriverHandle;
+use sail_execution::error::ExecutionResult;
 use tokio::sync::oneshot;
 use tokio::time::Instant;
 
@@ -34,6 +37,13 @@ pub enum SessionManagerEvent {
     ObserveState {
         observer: SessionManagerObserver,
     },
+    GetDriver {
+        driver_id: DriverId,
+        result: oneshot::Sender>,
+    },
+    Shutdown {
+        result: oneshot::Sender<()>,
+    },
 }
 
 pub struct SessionHistory {
@@ -49,6 +59,8 @@ impl SpanAssociation for SessionManagerEvent {
             SessionManagerEvent::SetSessionHistory { .. } => "SetSessionHistory",
             SessionManagerEvent::SetSessionFailure { .. } => "SetSessionFailure",
             SessionManagerEvent::ObserveState { .. } => "ObserveState",
+            SessionManagerEvent::GetDriver { .. } => "GetDriver",
+            SessionManagerEvent::Shutdown { .. } => "Shutdown",
         };
         name.into()
     }
@@ -76,7 +88,14 @@ impl SpanAssociation for SessionManagerEvent {
             | SessionManagerEvent::SetSessionFailure { session_id } => {
                 p.push((SpanAttribute::SESSION_ID, session_id.to_string()));
             }
-            SessionManagerEvent::ObserveState { observer: _ } => {}
+            SessionManagerEvent::GetDriver {
+                driver_id,
+                result: _,
+            } => {
+                p.push((SpanAttribute::CLUSTER_DRIVER_ID, driver_id.to_string()));
+            }
+            SessionManagerEvent::ObserveState { observer: _ }
+            | SessionManagerEvent::Shutdown { .. } => {}
         }
         p.into_iter().map(|(k, v)| (k.into(), v.into()))
     }
diff --git a/crates/sail-session/src/session_manager/mod.rs b/crates/sail-session/src/session_manager/mod.rs
index e3dc7cdc2..f66efb8de 100644
--- a/crates/sail-session/src/session_manager/mod.rs
+++ b/crates/sail-session/src/session_manager/mod.rs
@@ -4,16 +4,28 @@ mod options;
 mod session;
 
 use std::fmt;
+use std::sync::Arc;
+use std::time::Duration;
 
 use datafusion::prelude::SessionContext;
-use sail_server::actor::ActorHandle;
+use sail_common::actor::{ActorHandle, ActorSystem};
+use sail_common::config::{AppConfig, ExecutionMode};
+use sail_common::runtime::RuntimeHandle;
+use sail_execution::driver::{DriverGateway, DriverGatewayOptions};
 use tokio::sync::oneshot;
 
 use crate::error::{SessionError, SessionResult};
+use crate::session_factory::{
+    ServerSessionInfo, ServerSessionJobRunnerFactory, SessionFactory, SessionJobRunnerFactory,
+};
 pub(crate) use crate::session_manager::actor::SessionManagerActor;
 pub(crate) use crate::session_manager::event::SessionManagerEvent;
-pub use crate::session_manager::options::SessionManagerOptions;
+pub use crate::session_manager::options::{SessionManagerComponents, SessionManagerOptions};
 
+pub type ServerSessionFactoryFn =
+    fn(Arc, RuntimeHandle) -> Box>;
+
+#[derive(Clone)]
 pub struct SessionManager {
     handle: ActorHandle,
 }
@@ -25,9 +37,12 @@ impl fmt::Debug for SessionManager {
 }
 
 impl SessionManager {
-    pub fn try_new(options: SessionManagerOptions) -> SessionResult {
-        let system = options.system.clone();
-        let handle = system.lock()?.spawn::(options);
+    pub fn try_new(
+        options: SessionManagerOptions,
+        components: SessionManagerComponents,
+        system: &mut ActorSystem,
+    ) -> SessionResult {
+        let handle = system.spawn::((options, components));
         Ok(Self { handle })
     }
 
@@ -57,4 +72,54 @@ impl SessionManager {
         rx.await
             .map_err(|e| SessionError::internal(format!("failed to delete session: {e}")))?
     }
+
+    /// Shut down the session manager and all resources it owns.
+    pub async fn shutdown(&self) -> SessionResult<()> {
+        let (tx, rx) = oneshot::channel();
+        self.handle
+            .send(SessionManagerEvent::Shutdown { result: tx })
+            .await?;
+        rx.await.map_err(|e| {
+            SessionError::internal(format!("failed to shut down session manager: {e}"))
+        })?;
+        Ok(())
+    }
+}
+
+pub async fn create_session_manager(
+    config: Arc,
+    runtime: RuntimeHandle,
+    session_factory_fn: ServerSessionFactoryFn,
+    session_timeout: Duration,
+    system: &mut ActorSystem,
+) -> SessionResult {
+    let session_factory = session_factory_fn(config.clone(), runtime.clone());
+    let job_runner_factory = Box::new(ServerSessionJobRunnerFactory::new(
+        config.clone(),
+        runtime.clone(),
+    )) as Box;
+    let driver_gateway = if matches!(&config.mode, ExecutionMode::Local) {
+        None
+    } else {
+        Some(
+            DriverGateway::try_new(DriverGatewayOptions::new(&config))
+                .await
+                .map_err(|e| {
+                    SessionError::internal(format!("failed to create driver gateway: {e}"))
+                })?,
+        )
+    };
+    let options = SessionManagerOptions::new(runtime)
+        .with_session_timeout(session_timeout)
+        .with_options(
+            config
+                .raw()
+                .map_err(|e| SessionError::internal(e.to_string()))?,
+        );
+    let components = SessionManagerComponents {
+        session_factory,
+        job_runner_factory,
+        driver_gateway,
+    };
+    SessionManager::try_new(options, components, system)
 }
diff --git a/crates/sail-session/src/session_manager/options.rs b/crates/sail-session/src/session_manager/options.rs
index 099f09db2..4f452dac1 100644
--- a/crates/sail-session/src/session_manager/options.rs
+++ b/crates/sail-session/src/session_manager/options.rs
@@ -1,33 +1,30 @@
-use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use sail_common::runtime::RuntimeHandle;
-use sail_server::actor::ActorSystem;
+use sail_execution::driver::DriverGateway;
 
-use crate::session_factory::{ServerSessionInfo, SessionFactory};
+use crate::session_factory::{ServerSessionInfo, SessionFactory, SessionJobRunnerFactory};
 
 #[readonly::make]
 pub struct SessionManagerOptions {
     pub session_timeout: Duration,
     pub runtime: RuntimeHandle,
-    pub system: Arc>,
-    pub factory: Box Box> + Send>,
     /// The application configuration options as key-value pairs,
     /// used to populate the `system.session.options` table.
     pub options: Vec<(String, String)>,
 }
 
+pub struct SessionManagerComponents {
+    pub session_factory: Box>,
+    pub job_runner_factory: Box,
+    pub driver_gateway: Option,
+}
+
 impl SessionManagerOptions {
-    pub fn new(
-        runtime: RuntimeHandle,
-        system: Arc>,
-        factory: Box Box> + Send>,
-    ) -> Self {
+    pub fn new(runtime: RuntimeHandle) -> Self {
         Self {
             session_timeout: Duration::MAX,
             runtime,
-            system,
-            factory,
             options: Vec::new(),
         }
     }
diff --git a/crates/sail-session/src/session_manager/session.rs b/crates/sail-session/src/session_manager/session.rs
index 4aab89784..49ccb9970 100644
--- a/crates/sail-session/src/session_manager/session.rs
+++ b/crates/sail-session/src/session_manager/session.rs
@@ -10,6 +10,7 @@ use futures::FutureExt;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::job::JobService;
 use sail_common_datafusion::system::observable::{JobRunnerObserver, StateObservable};
+use sail_execution::DriverId;
 use tokio::sync::oneshot;
 
 use crate::session_manager::event::SessionHistory;
@@ -32,21 +33,20 @@ impl ServerSession {
     {
         let (tx, rx) = oneshot::channel();
         let observer = observer(tx);
-        if let ServerSessionState::Running { context } = &self.state {
-            if let Ok(service) = context.extension::() {
-                async move {
+        if let ServerSessionState::Running { context, .. } = &self.state {
+            match context.extension::() {
+                Ok(service) => async move {
                     service.runner().observe(observer).await;
                     rx.await
                         .map_err(|_| exec_datafusion_err!("failed to observe job runner"))?
                 }
-                .boxed()
-            } else {
-                async {
+                .boxed(),
+                _ => async {
                     Err(exec_datafusion_err!(
                         "job service not found in session context"
                     ))
                 }
-                .boxed()
+                .boxed(),
             }
         } else if let ServerSessionState::Deleted { history } = &self.state {
             let history = history.clone();
@@ -63,9 +63,16 @@ impl ServerSession {
 }
 
 pub enum ServerSessionState {
-    Running { context: SessionContext },
-    Deleting,
-    Deleted { history: Arc },
+    Running {
+        context: SessionContext,
+        driver_id: Option,
+    },
+    Deleting {
+        driver_id: Option,
+    },
+    Deleted {
+        history: Arc,
+    },
     Failed,
 }
 
@@ -73,7 +80,7 @@ impl ServerSessionState {
     pub fn status(&self) -> &'static str {
         match self {
             ServerSessionState::Running { .. } => "RUNNING",
-            ServerSessionState::Deleting => "DELETING",
+            ServerSessionState::Deleting { .. } => "DELETING",
             ServerSessionState::Deleted { .. } => "DELETED",
             ServerSessionState::Failed => "FAILED",
         }
diff --git a/crates/sail-spark-connect/Cargo.toml b/crates/sail-spark-connect/Cargo.toml
index ef0b7a90a..f2dbaf710 100644
--- a/crates/sail-spark-connect/Cargo.toml
+++ b/crates/sail-spark-connect/Cargo.toml
@@ -11,7 +11,6 @@ sail-cache = { path = "../sail-cache" }
 sail-common = { path = "../sail-common" }
 sail-common-datafusion = { path = "../sail-common-datafusion" }
 sail-data-source = { path = "../sail-data-source" }
-sail-server = { path = "../sail-server" }
 sail-plan = { path = "../sail-plan" }
 sail-python-udf = { path = "../sail-python-udf" }
 sail-sql-analyzer = { path = "../sail-sql-analyzer" }
diff --git a/crates/sail-spark-connect/build.rs b/crates/sail-spark-connect/build.rs
index 25e2b44de..7f301d8c8 100644
--- a/crates/sail-spark-connect/build.rs
+++ b/crates/sail-spark-connect/build.rs
@@ -1,6 +1,7 @@
 //! Build script that generates Rust from the .proto files.
 //! and also converts json to Rust constants.
 
+use std::collections::BTreeMap;
 use std::path::PathBuf;
 
 use prost_build::Config;
@@ -55,7 +56,7 @@ struct SparkConfig {
     entries: Vec,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 struct SparkConfigEntry {
     key: String,
@@ -74,28 +75,17 @@ struct SparkConfigEntry {
     // Reference: `org.apache.spark.internal.config.ConfigBuilder#withPrepended`
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 struct SparkConfigNotice {
     version: String,
     comment: String,
 }
 
-fn build_spark_config() -> Result<(), Box> {
-    println!("cargo:rerun-if-changed=data/spark_config.json");
-
-    let word_boundary = regex::Regex::new(r"(?P[a-z])(?P[A-Z])")?;
-    let key_const_name = |key: &str| -> String {
-        let key = word_boundary.replace_all(key, "${first}_${second}");
-        key.replace('.', "_").to_uppercase()
-    };
-
-    let mut config =
-        serde_json::from_str::(&std::fs::read_to_string("data/spark_config.json")?)?;
-
+fn apply_spark_config_overrides(config: &mut SparkConfig) {
     // TODO: remove these overrides
     config.entries.iter_mut().for_each(|entry| {
-        // Spark 4.1 changes the local relation cache threshold from 64 MB to 1 MB
+        // Spark 4.1+ changes the local relation cache threshold from 64 MB to 1 MB
         // which causes DataFrame creation more likely to fail since we do not support
         // caching local relations as artifacts yet.
         // Here we override the default value to 2^31-1 (2147483647, Integer.MAX_VALUE)
@@ -105,49 +95,52 @@ fn build_spark_config() -> Result<(), Box> {
         if entry.key == "spark.sql.session.localRelationCacheThreshold" {
             entry.default_value = Some("2147483647".to_string())
         }
-        // Spark 4.1 enables safe conversion to Arrow by default, but we turn it off
+        // Spark 4.1+ enables safe conversion to Arrow by default, but we turn it off
         // to avoid a few PySpark test failures caused by datetime conversions.
         // More investigation is needed here.
         if entry.key == "spark.sql.execution.pandas.convertToArrowArraySafely" {
             entry.default_value = Some("false".to_string())
         }
+        // Spark 4.1+ allows plan compression which we do not support yet, so we turn it off.
+        if entry.key == "spark.connect.session.planCompression.threshold" {
+            entry.default_value = Some("-1".to_string())
+        }
     });
+}
 
-    // The JVM Spark Connect client reads spark.connect.session.planCompression.threshold
-    // to decide whether to compress plans before sending them to the server.
-    // This key is not part of Spark's SQL config (spark_config.json) but the client
-    // expects it to exist. Without it, the client logs a NoSuchElementException warning.
-    if !config
-        .entries
-        .iter()
-        .any(|entry| entry.key == "spark.connect.session.planCompression.threshold")
-    {
-        config.entries.push(SparkConfigEntry {
-            key: "spark.connect.session.planCompression.threshold".to_string(),
-            doc: "Minimum plan size in bytes before the client attempts to compress it."
-                .to_string(),
-            default_value: Some("1048576".to_string()),
-            alternatives: Vec::new(),
-            fallback: None,
-            is_static: false,
-            deprecated: None,
-            removed: None,
-        });
+fn build_spark_config(versions: &[&str]) -> Result<(), Box> {
+    let word_boundary = regex::Regex::new(r"(?P[a-z])(?P[A-Z])")?;
+    let key_const_name = |key: &str| -> String {
+        let key = word_boundary.replace_all(key, "${first}_${second}");
+        key.replace('.', "_").to_uppercase()
+    };
+
+    let mut configs = Vec::with_capacity(versions.len());
+    for &version in versions {
+        let path = format!("data/config/spark-{version}.json");
+        println!("cargo:rerun-if-changed={path}");
+        let mut config = serde_json::from_str::(&std::fs::read_to_string(&path)?)?;
+        apply_spark_config_overrides(&mut config);
+        configs.push((version, config));
     }
 
-    let keys = config
-        .entries
-        .iter()
-        .map(|entry| {
-            let key = &entry.key;
-            let doc = if !entry.doc.is_empty() {
-                &entry.doc
+    let mut entries_by_key: BTreeMap)>> = BTreeMap::new();
+    for (version, config) in &configs {
+        for entry in &config.entries {
+            let variants = entries_by_key.entry(entry.key.clone()).or_default();
+            if let Some((_, versions)) = variants.iter_mut().find(|(x, _)| x == entry) {
+                versions.push(*version);
             } else {
-                "(Missing documentation)"
-            };
+                variants.push((entry.clone(), vec![*version]));
+            }
+        }
+    }
+
+    let keys = entries_by_key
+        .keys()
+        .map(|key| {
             let const_name = format_ident!("{}", key_const_name(key));
             quote! {
-                #[doc = #doc]
                 pub const #const_name: &str = #key;
             }
         })
@@ -167,35 +160,83 @@ fn build_spark_config() -> Result<(), Box> {
         }
     };
 
-    let entries = config
-        .entries
+    let entry_to_token_stream = |entry: &SparkConfigEntry| {
+        let key_const_name = format_ident!("{}", key_const_name(&entry.key));
+        let default_value = match &entry.default_value {
+            None => quote! { None },
+            Some(x) => quote! { Some(#x) },
+        };
+        let alternatives = &entry.alternatives;
+        let fallback = match &entry.fallback {
+            None => quote! { None },
+            Some(x) => {
+                quote! { Some(#x) }
+            }
+        };
+        let is_static = entry.is_static;
+        let deprecated = notice_to_token_stream(&entry.deprecated);
+        let removed = notice_to_token_stream(&entry.removed);
+        quote! {
+            SparkConfigEntry {
+                key: SparkConfigKey::#key_const_name,
+                default_value: #default_value,
+                alternatives: &[#(#alternatives),*],
+                fallback: #fallback,
+                is_static: #is_static,
+                deprecated: #deprecated,
+                removed: #removed,
+            }
+        }
+    };
+
+    let mut entries = Vec::new();
+    for variants in entries_by_key.values_mut() {
+        variants.sort_by_key(|(_, versions)| versions[0]);
+        entries.extend(variants.iter().cloned());
+    }
+
+    let mut entry_names = BTreeMap::new();
+    let mut entry_definitions = Vec::with_capacity(entries.len());
+    for (entry, versions) in &entries {
+        let name = format_ident!(
+            "_SPARK_CONFIG_ENTRY_V{}_{}",
+            versions[0].replace('.', "_"),
+            key_const_name(&entry.key),
+        );
+        for version in versions {
+            entry_names.insert(((*version).to_string(), entry.key.clone()), name.clone());
+        }
+        let doc = if entry.doc.is_empty() {
+            "(Missing documentation)"
+        } else {
+            &entry.doc
+        };
+        let entry = entry_to_token_stream(entry);
+        entry_definitions.push(quote! {
+            #[doc = #doc]
+            static #name: SparkConfigEntry<'static> = #entry;
+        });
+    }
+
+    let config_maps = configs
         .iter()
-        .map(|entry| {
-            let key = &entry.key;
-            let default_value = match &entry.default_value {
-                None => quote! { None },
-                Some(x) => quote! { Some(#x) },
-            };
-            let alternatives = &entry.alternatives;
-            let fallback = match &entry.fallback {
-                None => quote! { None },
-                Some(x) => {
-                    quote! { Some(#x) }
-                }
-            };
-            let is_static = entry.is_static;
-            let deprecated = notice_to_token_stream(&entry.deprecated);
-            let removed = notice_to_token_stream(&entry.removed);
+        .map(|(version, config)| {
+            let name = format_ident!("SPARK_CONFIG_V{}", version.replace('.', "_"));
+            let macro_name = format_ident!("spark_config_map_v{}", version.replace('.', "_"));
+            let map_entries = config.entries.iter().map(|entry| {
+                let key = &entry.key;
+                let entry_name = &entry_names[&((*version).to_string(), key.clone())];
+                quote! { #key => &#entry_name, }
+            });
+            // We define the map in a separate macro to avoid slowing down the IDE
+            // when previewing the definitions of the versioned config maps.
             quote! {
-                #key => SparkConfigEntry {
-                    key: #key,
-                    default_value: #default_value,
-                    alternatives: &[#(#alternatives),*],
-                    fallback: #fallback,
-                    is_static: #is_static,
-                    deprecated: #deprecated,
-                    removed: #removed,
-                },
+                macro_rules! #macro_name {
+                    () => { phf::phf_map! { #(#map_entries)* } }
+                }
+
+                pub static #name: phf::Map<&'static str, &'static SparkConfigEntry<'static>> =
+                    #macro_name!();
             }
         })
         .collect::>();
@@ -218,15 +259,15 @@ fn build_spark_config() -> Result<(), Box> {
             pub comment: &'a str,
         }
 
-        #(#keys)*
+        pub struct SparkConfigKey;
 
-        // We define the map in a separate macro to avoid slowing down the IDE
-        // when previewing the definition of `SPARK_CONFIG`.
-        macro_rules! spark_config_map {
-            () => { phf::phf_map! { #(#entries)* } }
+        impl SparkConfigKey {
+            #(#keys)*
         }
 
-        pub static SPARK_CONFIG: phf::Map<&'static str, SparkConfigEntry<'static>> = spark_config_map!();
+        #(#entry_definitions)*
+
+        #(#config_maps)*
     };
 
     let out_dir = PathBuf::from(std::env::var("OUT_DIR")?);
@@ -240,6 +281,6 @@ fn build_spark_config() -> Result<(), Box> {
 fn main() -> Result<(), Box> {
     println!("cargo:rerun-if-changed=build.rs");
     build_proto()?;
-    build_spark_config()?;
+    build_spark_config(&["3.5", "4.0", "4.1", "4.2"])?;
     Ok(())
 }
diff --git a/crates/sail-spark-connect/data/config/spark-3.5.json b/crates/sail-spark-connect/data/config/spark-3.5.json
new file mode 100644
index 000000000..e312b776f
--- /dev/null
+++ b/crates/sail-spark-connect/data/config/spark-3.5.json
@@ -0,0 +1,2584 @@
+{
+  "sparkVersion": "3.5.9",
+  "comment": "This is a generated file. Do not edit it directly.",
+  "entries": [
+    {
+      "key": "spark.connect.copyFromLocalToFs.allowDestLocal",
+      "doc": "\nAllow `spark.copyFromLocalToFs` destination to be local file system\n path on spark driver node when\n`spark.connect.copyFromLocalToFs.allowDestLocal` is true.\nThis will allow user to overwrite arbitrary file on spark\ndriver node we should only enable it for testing purpose.\n",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.abandonedTombstonesSize",
+      "doc": "Maximum size of the cache of abandoned executions.",
+      "defaultValue": "10000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.detachedTimeout",
+      "doc": "Timeout after which executions without an attached RPC will be removed.",
+      "defaultValue": "5m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.maintenanceInterval",
+      "doc": "Interval at which execution manager will search for abandoned executions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.enabled",
+      "doc": "Enables reattachable execution on the server. If disabled and a client requests it, non-reattachable execution will follow and should run until query completion. This will work, unless there is a GRPC stream error, in which case the client will discover that execution is not reattachable when trying to reattach fails.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.observerRetryBufferSize",
+      "doc": "For reattachable execution, the total size of responses that were already sent to be kept in the buffer in case of connection error and client needing to retry. Set 0 to don't buffer anything (even last sent response).With any value greater than 0, the last sent response will always be buffered.",
+      "defaultValue": "10m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamDuration",
+      "doc": "For reattachable execution, after this amount of time the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "2m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamSize",
+      "doc": "For reattachable execution, after total responses size exceeds this value, the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "1g",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.command.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.CommandPlugin to support custom\nCommand types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.expression.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.ExpressionPlugin to support custom\nExpression types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.relation.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.RelationPlugin to support custom\nRelation types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.arrow.maxBatchSize",
+      "doc": "When using Apache Arrow, limit the maximum size of one arrow batch, in bytes unless otherwise specified, that can be sent from server side to client side. Currently, we conservatively use 70% of it because the size is not accurate but estimated.",
+      "defaultValue": "4194304b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.port",
+      "doc": "The port for Spark Connect server to bind.",
+      "defaultValue": "15002",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.interceptor.classes",
+      "doc": "Comma separated list of class names that must implement the io.grpc.ServerInterceptor interface.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.marshallerRecursionLimit",
+      "doc": "\nSets the recursion limit to grpc protobuf messages.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxInboundMessageSize",
+      "doc": "Sets the maximum inbound message in bytes size for the gRPC requests.Requests with a larger payload will fail.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.jvmStacktrace.maxSize",
+      "doc": "\nSets the maximum stack trace size to display when\n`spark.sql.pyspark.jvmStacktrace.enabled` is true.\n",
+      "defaultValue": "2048",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.adaptive.advisoryPartitionSizeInBytes",
+      "doc": "The advisory size in bytes of the shuffle partition during adaptive optimization (when spark.sql.adaptive.enabled is true). It takes effect when Spark coalesces small shuffle partitions or splits skewed shuffle partition.",
+      "fallback": "spark.sql.adaptive.shuffle.targetPostShuffleInputSize"
+    },
+    {
+      "key": "spark.sql.adaptive.applyFinalStageShuffleOptimizations",
+      "doc": "Configures whether adaptive query execution (if enabled) should apply shuffle coalescing and local shuffle read optimization for the final query stage.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.autoBroadcastJoinThreshold",
+      "doc": "Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join. By setting this value to -1 broadcasting can be disabled. The default value is same with spark.sql.autoBroadcastJoinThreshold. Note that, this config is used only in adaptive framework."
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark will coalesce contiguous shuffle partitions according to the target size (specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes'), to avoid too many small tasks.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.initialPartitionNum",
+      "doc": "The initial number of shuffle partitions before coalescing. If not set, it equals to spark.sql.shuffle.partitions. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true."
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.minPartitionNum",
+      "doc": "(deprecated) The suggested (not guaranteed) minimum number of shuffle partitions after coalescing. If not set, the default value is the default parallelism of the Spark cluster. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true.",
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use 'spark.sql.adaptive.coalescePartitions.minPartitionSize' instead."
+      }
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.minPartitionSize",
+      "doc": "The minimum size of shuffle partitions after coalescing. This is useful when the adaptively calculated target size is too small during partition coalescing.",
+      "defaultValue": "1MB"
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.parallelismFirst",
+      "doc": "When true, Spark does not respect the target size specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes' (default 64MB) when coalescing contiguous shuffle partitions, but adaptively calculate the target size according to the default parallelism of the Spark cluster. The calculated size is usually smaller than the configured target size. This is to maximize the parallelism and avoid performance regression when enabling adaptive query execution. It's recommended to set this config to false and respect the configured target size.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.customCostEvaluatorClass",
+      "doc": "The custom cost evaluator class to be used for adaptive execution. If not being set, Spark will use its own SimpleCostEvaluator by default."
+    },
+    {
+      "key": "spark.sql.adaptive.enabled",
+      "doc": "When true, enable adaptive query execution, which re-optimizes the query plan in the middle of query execution, based on accurate runtime statistics.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.fetchShuffleBlocksInBatch",
+      "doc": "Whether to fetch the contiguous shuffle blocks in batch. Instead of fetching blocks one by one, fetching contiguous shuffle blocks for the same map task in batch can reduce IO and improve performance. Note, multiple contiguous blocks exist in single fetch request only happen when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true. This feature also depends on a relocatable serializer, the concatenation support codec in use, the new version shuffle fetch protocol and io encryption is disabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.forceApply",
+      "doc": "Adaptive query execution is skipped when the query does not have exchanges or sub-queries. By setting this config to true (together with 'spark.sql.adaptive.enabled' set to true), Spark will force apply adaptive query execution for all supported queries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.adaptive.forceOptimizeSkewedJoin",
+      "doc": "When true, force enable OptimizeSkewedJoin even if it introduces extra shuffle.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.adaptive.localShuffleReader.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark tries to use local shuffle reader to read the shuffle data when the shuffle partitioning is not needed, for example, after converting sort-merge join to broadcast-hash join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.logLevel",
+      "doc": "Configures the log level for adaptive execution logging of plan changes. The value can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.",
+      "defaultValue": "debug"
+    },
+    {
+      "key": "spark.sql.adaptive.maxShuffledHashJoinLocalMapThreshold",
+      "doc": "Configures the maximum size in bytes per partition that can be allowed to build local hash map. If this value is not smaller than spark.sql.adaptive.advisoryPartitionSizeInBytes and all the partition size are not larger than this config, join selection prefer to use shuffled hash join instead of sort merge join regardless of the value of spark.sql.join.preferSortMergeJoin.",
+      "defaultValue": "0b"
+    },
+    {
+      "key": "spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin",
+      "doc": "The relation with a non-empty partition ratio lower than this config will not be considered as the build side of a broadcast-hash join in adaptive execution regardless of its size.This configuration only has an effect when 'spark.sql.adaptive.enabled' is true.",
+      "defaultValue": "0.2"
+    },
+    {
+      "key": "spark.sql.adaptive.optimizeSkewsInRebalancePartitions.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark will optimize the skewed shuffle partitions in RebalancePartitions and split them to smaller ones according to the target size (specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes'), to avoid data skew.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.optimizer.excludedRules",
+      "doc": "Configures a list of rules to be disabled in the adaptive optimizer, in which the rules are specified by their rule names and separated by comma. The optimizer will log the rules that have indeed been excluded."
+    },
+    {
+      "key": "spark.sql.adaptive.rebalancePartitionsSmallPartitionFactor",
+      "doc": "A partition will be merged during splitting if its size is small than this factor multiply spark.sql.adaptive.advisoryPartitionSizeInBytes.",
+      "defaultValue": "0.2"
+    },
+    {
+      "key": "spark.sql.adaptive.shuffle.targetPostShuffleInputSize",
+      "doc": "(Deprecated since Spark 3.0)",
+      "defaultValue": "64MB",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.adaptive.advisoryPartitionSizeInBytes' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark dynamically handles skew in shuffled join (sort-merge and shuffled hash) by splitting (and replicating if needed) skewed partitions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.skewedPartitionFactor",
+      "doc": "A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than 'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'",
+      "defaultValue": "5.0"
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes",
+      "doc": "A partition is considered as skewed if its size in bytes is larger than this threshold and also larger than 'spark.sql.adaptive.skewJoin.skewedPartitionFactor' multiplying the median partition size. Ideally this config should be set larger than 'spark.sql.adaptive.advisoryPartitionSizeInBytes'.",
+      "defaultValue": "256MB"
+    },
+    {
+      "key": "spark.sql.addPartitionInBatch.size",
+      "doc": "The number of partitions to be handled in one turn when use `AlterTableAddPartitionCommand` or `RepairTableCommand` to add partitions into table. The smaller batch size is, the less memory is required for the real handler, e.g. Hive Metastore.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.allowNamedFunctionArguments",
+      "doc": "If true, Spark will turn on support for named parameters for all functions that has it implemented.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.canonicalization.multiCommutativeOpMemoryOptThreshold",
+      "doc": "The minimum number of operands in a commutative expression tree to invoke the MultiCommutativeOp memory optimization during canonicalization.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.analyzer.failAmbiguousSelfJoin",
+      "doc": "When true, fail the Dataset query if it contains ambiguous self-join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.maxIterations",
+      "doc": "The max number of iterations the analyzer runs.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.ansi.doubleQuotedIdentifiers",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, Spark SQL reads literals enclosed in double quoted (\") as identifiers. When false they are read as string literals.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.enabled",
+      "doc": "When true, Spark SQL uses an ANSI compliant dialect instead of being Hive compliant. For example, Spark will throw an exception at runtime instead of returning null results when the inputs to a SQL operator/function are invalid.For full details of this dialect, you can find them in the section \"ANSI Compliance\" of Spark's documentation. Some ANSI dialect features may be not from the ANSI SQL standard directly, but their behaviors align with ANSI SQL's style",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.enforceReservedKeywords",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, the Spark SQL parser enforces the ANSI reserved keywords and forbids SQL queries that use reserved keywords as alias names and/or identifiers for table, view, function, etc.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.relationPrecedence",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, JOIN takes precedence over comma when combining relation. For example, `t1, t2 JOIN t3` should result to `t1 X (t2 X t3)`. If the config is false, the result is `(t1 X t2) X t3`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.strictIndexOperator",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.4.0",
+        "comment": "This was an internal configuration. It is not needed anymore since Spark SQL always returns null when getting a map value with a non-existing key. See SPARK-40066 for more details."
+      }
+    },
+    {
+      "key": "spark.sql.autoBroadcastJoinThreshold",
+      "doc": "Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join.  By setting this value to -1 broadcasting can be disabled. Note that currently statistics are only supported for Hive Metastore tables where the command `ANALYZE TABLE  COMPUTE STATISTICS noscan` has been run, and file-based data source tables where the statistics are computed directly on the files of data.",
+      "defaultValue": "10MB"
+    },
+    {
+      "key": "spark.sql.avro.compression.codec",
+      "doc": "Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate, snappy, bzip2, xz and zstandard. Default codec is snappy.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.avro.datetimeRebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files. When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. When EXCEPTION, which is the default, Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Avro files is unknown.",
+      "defaultValue": "EXCEPTION",
+      "alternatives": [
+        "spark.sql.legacy.avro.datetimeRebaseModeInRead"
+      ]
+    },
+    {
+      "key": "spark.sql.avro.datetimeRebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files. When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. When EXCEPTION, which is the default, Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars.",
+      "defaultValue": "EXCEPTION",
+      "alternatives": [
+        "spark.sql.legacy.avro.datetimeRebaseModeInWrite"
+      ]
+    },
+    {
+      "key": "spark.sql.avro.deflate.level",
+      "doc": "Compression level for the deflate codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level in the current implementation.",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.avro.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to Avro datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.broadcastExchange.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to fetch and broadcast the table. If we encounter memory issue like frequently full GC or OOM when broadcast table we can decrease this number in order to reduce memory usage. Notice the number should be carefully chosen since decreasing parallelism might cause longer waiting for other broadcasting. Also, increasing parallelism may cause memory problem.",
+      "defaultValue": "128",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.broadcastTimeout",
+      "doc": "Timeout in seconds for the broadcast wait time in broadcast joins.",
+      "defaultValue": "300"
+    },
+    {
+      "key": "spark.sql.bucketing.coalesceBucketsInJoin.enabled",
+      "doc": "When true, if two bucketed tables with the different number of buckets are joined, the side with a bigger number of buckets will be coalesced to have the same number of buckets as the other side. Bigger number of buckets is divisible by the smaller number of buckets. Bucket coalescing is applied to sort-merge joins and shuffled hash join. Note: Coalescing bucketed table can avoid unnecessary shuffling in join, but it also reduces parallelism and could possibly cause OOM for shuffled hash join.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.bucketing.coalesceBucketsInJoin.maxBucketRatio",
+      "doc": "The ratio of the number of two buckets being coalesced should be less than or equal to this value for bucket coalescing to be applied. This configuration only has an effect when 'spark.sql.bucketing.coalesceBucketsInJoin.enabled' is set to true.",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.cache.serializer",
+      "doc": "The name of a class that implements org.apache.spark.sql.columnar.CachedBatchSerializer. It will be used to translate SQL data into a format that can more efficiently be cached. The underlying API is subject to change so use with caution. Multiple classes cannot be specified. The class must have a no-arg constructor.",
+      "defaultValue": "org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the cartesian product operator",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by cartesian product operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.caseSensitive",
+      "doc": "Whether the query analyzer should be case sensitive or not. Default to case insensitive. It is highly discouraged to turn on case sensitive mode.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.catalog.spark_catalog",
+      "doc": "A catalog implementation that will be used as the v2 interface to Spark's built-in v1 catalog: spark_catalog. This catalog shares its identifier namespace with the spark_catalog and must be consistent with it; for example, if a table can be loaded by the spark_catalog, this catalog must also return the table metadata. To delegate operations to the spark_catalog, implementations can extend 'CatalogExtension'."
+    },
+    {
+      "key": "spark.sql.catalog.spark_catalog.defaultDatabase",
+      "doc": "The default database for session catalog.",
+      "defaultValue": "default",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.catalogImplementation",
+      "doc": "",
+      "defaultValue": "in-memory",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.cbo.enabled",
+      "doc": "Enables CBO for estimation of plan statistics when set true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.card.weight",
+      "doc": "The weight of the ratio of cardinalities (number of rows) in the cost comparison function. The ratio of sizes in bytes has weight 1 - this value. The weighted geometric mean of these ratios is used to decide which of the candidate plans will be chosen by the CBO.",
+      "defaultValue": "0.7"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.dp.star.filter",
+      "doc": "Applies star-join filter heuristics to cost based join enumeration.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.dp.threshold",
+      "doc": "The maximum number of joined nodes allowed in the dynamic programming algorithm.",
+      "defaultValue": "12"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.enabled",
+      "doc": "Enables join reorder in CBO.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.planStats.enabled",
+      "doc": "When true, the logical plan will fetch row counts and column statistics from catalog.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.starJoinFTRatio",
+      "doc": "Specifies the upper limit of the ratio between the largest fact tables for a star join to be considered. ",
+      "defaultValue": "0.9"
+    },
+    {
+      "key": "spark.sql.cbo.starSchemaDetection",
+      "doc": "When true, it enables join reordering based on star schema detection. ",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.charAsVarchar",
+      "doc": "When true, Spark replaces CHAR type with VARCHAR type in CREATE/REPLACE/ALTER TABLE commands, so that newly created/updated tables will not have CHAR type columns/fields. Existing tables with CHAR type columns/fields are not affected by this config.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.chunkBase64String.enabled",
+      "doc": "Whether to truncate string generated by the `Base64` function. When true, base64 strings generated by the base64 function are chunked into lines of at most 76 characters. When false, the base64 strings are not chunked.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.cli.print.header",
+      "doc": "When set to true, spark-sql CLI prints the names of the columns in query output.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.fastHashMap.capacityBit",
+      "doc": "Capacity for the max number of rows to be held in memory by the fast hash aggregate product operator. The bit is not for actual value, but the actual numBuckets is determined by loadFactor (e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.twolevel.enabled",
+      "doc": "Enable two-level aggregate hash map. When enabled, records will first be inserted/looked-up at a 1st-level, small, fast map, and then fallback to a 2nd-level, larger, slower map when 1st level is full or keys cannot be found. When disabled, records go directly to the 2nd level.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.twolevel.partialOnly",
+      "doc": "Enable two-level aggregate hash map for partial aggregate only, because final aggregate might get more distinct keys compared to partial aggregate. Overhead of looking up 1st-level map might dominate when having a lot of distinct keys.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.vectorized.enable",
+      "doc": "Enable vectorized aggregate hash map. This is for testing/benchmarking only.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.sortAggregate.enabled",
+      "doc": "When true, enable code-gen for sort aggregate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.splitAggregateFunc.enabled",
+      "doc": "When true, the code generator would split aggregate code into individual methods instead of a single big method. This can be used to avoid oversized function that can miss the opportunity of JIT optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.cache.maxEntries",
+      "doc": "When nonzero, enable caching of generated classes for operators and expressions. All jobs share the cache that can use up to the specified number for generated classes.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.codegen.comments",
+      "doc": "When true, put comment in the generated code. Since computing huge comments can be extremely expensive in certain cases, such as deeply-nested expressions which operate over inputs with wide schemas, default is false.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.codegen.factoryMode",
+      "doc": "This config determines the fallback behavior of several codegen generators during tests. `FALLBACK` means trying codegen first and then falling back to interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. `NO_CODEGEN` skips codegen and goes interpreted path always. Note that this configuration is only for the internal usage, and NOT supposed to be set by end users.",
+      "defaultValue": "FALLBACK"
+    },
+    {
+      "key": "spark.sql.codegen.fallback",
+      "doc": "When true, (whole stage) codegen could be temporary disabled for the part of query that fail to compile generated code",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.hugeMethodLimit",
+      "doc": "The maximum bytecode size of a single compiled Java function generated by whole-stage codegen. When the compiled function exceeds this threshold, the whole-stage codegen is deactivated for this subtree of the current query plan. The default value is 65535, which is the largest bytecode size possible for a valid Java method. When running on HotSpot, it may be preferable to set the value to 8000 to match HotSpot's implementation.",
+      "defaultValue": "65535"
+    },
+    {
+      "key": "spark.sql.codegen.join.buildSideOuterShuffledHashJoin.enabled",
+      "doc": "When true, enable code-gen for an OUTER shuffled hash join where outer side is the build side.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.existenceSortMergeJoin.enabled",
+      "doc": "When true, enable code-gen for Existence sort merge join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.fullOuterShuffledHashJoin.enabled",
+      "doc": "When true, enable code-gen for FULL OUTER shuffled hash join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.fullOuterSortMergeJoin.enabled",
+      "doc": "When true, enable code-gen for FULL OUTER sort merge join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.logging.maxLines",
+      "doc": "The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.codegen.maxFields",
+      "doc": "The maximum number of fields (including nested fields) that will be supported before deactivating whole-stage codegen.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.codegen.methodSplitThreshold",
+      "doc": "The threshold of source-code splitting in the codegen. When the number of characters in a single Java function (without comment) exceeds the threshold, the function will be automatically split to multiple smaller ones. We cannot know how many bytecode will be generated, so use the code length as metric. When running on HotSpot, a function's bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not be too small, otherwise there will be many function calls.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.codegen.splitConsumeFuncByOperator",
+      "doc": "When true, whole stage codegen would put the logic of consuming rows of each physical operator into individual methods, instead of a single big method. This can be used to avoid oversized function that can miss the opportunity of JIT optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.useIdInClassName",
+      "doc": "When true, embed the (whole-stage) codegen stage ID into the class name of the generated class as a suffix",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.wholeStage",
+      "doc": "When true, the whole stage (of multiple operators) will be compiled into single java method.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.columnNameOfCorruptRecord",
+      "doc": "The name of internal column for storing raw/un-parsed JSON and CSV records that fail to parse.",
+      "defaultValue": "_corrupt_record"
+    },
+    {
+      "key": "spark.sql.columnVector.offheap.enabled",
+      "doc": "When true, use OffHeapColumnVector in ColumnarBatch.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedSessions",
+      "doc": "The number of client sessions kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedStatements",
+      "doc": "The number of statements kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.constraintPropagation.enabled",
+      "doc": "When true, the query optimizer will infer and propagate data constraints in the query plan to optimize them. Constraint propagation can sometimes be computationally expensive for certain kinds of query plans (such as those with a large number of predicates and aliases) which might negatively impact overall runtime.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.crossJoin.enabled",
+      "doc": "When false, we will throw an error if a query contains a cartesian product without explicit CROSS JOIN syntax.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to CSV datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.parser.columnPruning.enabled",
+      "doc": "If it is set to true, column names of the requested schema are passed to CSV parser. Other column values can be ignored during parsing even if they are malformed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.parser.inputBufferSize",
+      "doc": "If it is set, it configures the buffer size of CSV input during parsing. It is the same as inputBufferSize option in CSV which has a higher priority. Note that this is a workaround for the parsing library's regression, and this configuration is internal and supposed to be removed in the near future."
+    },
+    {
+      "key": "spark.sql.datetime.java8API.enabled",
+      "doc": "If the configuration property is set to true, java.time.Instant and java.time.LocalDate classes of Java 8 API are used as external types for Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp and java.sql.Date are used for the same purpose.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.debug",
+      "doc": "Only used for internal debugging. Not all functions are supported when it is enabled.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.debug.maxToStringFields",
+      "doc": "Maximum number of fields of sequence-like entries can be converted to strings in debug output. Any elements beyond the limit will be dropped and replaced by a \"... N more fields\" placeholder.",
+      "defaultValue": "25"
+    },
+    {
+      "key": "spark.sql.decimalOperations.allowPrecisionLoss",
+      "doc": "When true (default), establishing the result type of an arithmetic operation happens according to Hive behavior and SQL ANSI 2011 specification, i.e. rounding the decimal part of the result if an exact representation is not possible. Otherwise, NULL is returned in those cases, as previously.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultCatalog",
+      "doc": "Name of the default catalog. This will be the current catalog if users have not explicitly set the current catalog yet.",
+      "defaultValue": "spark_catalog"
+    },
+    {
+      "key": "spark.sql.defaultColumn.allowedProviders",
+      "doc": "List of table providers wherein SQL commands are permitted to assign DEFAULT column values. Comma-separated list, whitespace ignored, case-insensitive. If an asterisk appears after any table provider in this list, any command may assign DEFAULT column except `ALTER TABLE ... ADD COLUMN`. Otherwise, if no asterisk appears, all commands are permitted. This is useful because in order for such `ALTER TABLE ... ADD COLUMN` commands to work, the target data source must include support for substituting in the provided values when the corresponding fields are not present in storage.",
+      "defaultValue": "csv,json,orc,parquet"
+    },
+    {
+      "key": "spark.sql.defaultColumn.enabled",
+      "doc": "When true, allow CREATE TABLE, REPLACE TABLE, and ALTER COLUMN statements to set or update default values for specific columns. Following INSERT, MERGE, and UPDATE statements may then omit these values and their values will be injected automatically instead.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultColumn.useNullsForMissingDefaultValues",
+      "doc": "When true, and DEFAULT columns are enabled, allow INSERT INTO commands with user-specified lists of fewer columns than the target table to behave as if they had specified DEFAULT for all remaining columns instead, in order.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultSizeInBytes",
+      "doc": "The default table size used in query planning. By default, it is set to Long.MaxValue which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. That is to say by default the optimizer will not choose to broadcast a table unless it knows for sure its size is small enough.",
+      "defaultValue": "9223372036854775807b"
+    },
+    {
+      "key": "spark.sql.defaultUrlStreamHandlerFactory.enabled",
+      "doc": "When true, register Hadoop's FsUrlStreamHandlerFactory to support ADD JAR against HDFS locations. It should be disabled when a different stream protocol handler should be registered to support a particular protocol type, or if Hadoop's FsUrlStreamHandlerFactory conflicts with other protocol types such as `http` or `https`. See also SPARK-25694 and HADOOP-14598.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.error.messageFormat",
+      "doc": "When PRETTY, the error message consists of textual representation of error class, message and query context. The MINIMAL and STANDARD formats are pretty JSON formats where STANDARD includes an additional JSON field `message`. This configuration property influences on error messages of Thrift Server and SQL CLI while running queries.",
+      "defaultValue": "PRETTY"
+    },
+    {
+      "key": "spark.sql.event.truncate.length",
+      "doc": "Threshold of SQL length beyond which it will be truncated before adding to event. Defaults to no truncation. If set to 0, callsite will be logged instead.",
+      "defaultValue": "2147483647",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.exchange.reuse",
+      "doc": "When true, the planner will try to find out duplicated exchanges and re-use them.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.arrow.enabled",
+      "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.execution.arrow.pyspark.enabled' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.execution.arrow.fallback.enabled",
+      "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.fallback.enabled'.)",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.execution.arrow.pyspark.fallback.enabled' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.execution.arrow.localRelationThreshold",
+      "doc": "When converting Arrow batches to Spark DataFrame, local collections are used in the driver side if the byte size of Arrow batches is smaller than this threshold. Otherwise, the Arrow batches are sent and deserialized to Spark internal rows in the executors.",
+      "defaultValue": "48MB"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxRecordsPerBatch",
+      "doc": "When using Apache Arrow, limit the maximum number of records that can be written to a single ArrowRecordBatch in memory. This configuration is not effective for the grouping API such as DataFrame(.cogroup).groupby.applyInPandas because each group becomes each ArrowRecordBatch. If set to zero or negative there is no limit.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.enabled",
+      "doc": "When true, make use of Apache Arrow for columnar data transfers in PySpark. This optimization applies to: 1. pyspark.sql.DataFrame.toPandas. 2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame or a NumPy ndarray. The following data type is unsupported: ArrayType of TimestampType.",
+      "fallback": "spark.sql.execution.arrow.enabled"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.fallback.enabled",
+      "doc": "When true, optimizations enabled by 'spark.sql.execution.arrow.pyspark.enabled' will fallback automatically to non-optimized implementations if an error occurs.",
+      "fallback": "spark.sql.execution.arrow.fallback.enabled"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.selfDestruct.enabled",
+      "doc": "(Experimental) When true, make use of Apache Arrow's self-destruct and split-blocks options for columnar data transfers in PySpark, when converting from Arrow to Pandas. This reduces memory usage at the cost of some CPU time. This optimization applies to: pyspark.sql.DataFrame.toPandas when 'spark.sql.execution.arrow.pyspark.enabled' is set.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.sparkr.enabled",
+      "doc": "When true, make use of Apache Arrow for columnar data transfers in SparkR. This optimization applies to: 1. createDataFrame when its input is an R DataFrame 2. collect 3. dapply 4. gapply The following data types are unsupported: FloatType, BinaryType, ArrayType, StructType and MapType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.useLargeVarTypes",
+      "doc": "When using Apache Arrow, use large variable width vectors for string and binary types. Regular string and binary types have a 2GiB limit for a column in a single record batch. Large variable types remove this limitation at the cost of higher memory usage per value. Note that this only works for DataFrame.mapInArrow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.broadcastHashJoin.outputPartitioningExpandLimit",
+      "doc": "The maximum number of partitionings that a HashPartitioning can be expanded to. This configuration is applicable only for BroadcastHashJoin inner joins and can be set to '0' to disable this feature.",
+      "defaultValue": "8"
+    },
+    {
+      "key": "spark.sql.execution.fastFailOnFileFormatOutput",
+      "doc": "Whether to fast fail task execution when writing output to FileFormat datasource. If this is enabled, in `FileFormatWriter` we will catch `FileAlreadyExistsException` and fast fail output task without further task retry. Only enabling this if you know the `FileAlreadyExistsException` of the output task is unrecoverable, i.e., further task attempts won't be able to success. If the `FileAlreadyExistsException` might be recoverable, you should keep this as disabled and let Spark to retry output tasks. This is disabled by default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pandas.convertToArrowArraySafely",
+      "doc": "When true, Arrow will perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. When false, disabling Arrow's type check and do type conversions anyway. This config only works for Arrow 0.11.0+.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pandas.respectSessionTimeZone",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "The non-default behavior is considered as a bug, see SPARK-22395. The config was deprecated since Spark 2.3."
+      }
+    },
+    {
+      "key": "spark.sql.execution.pandas.structHandlingMode",
+      "doc": "The conversion mode of struct type when creating pandas DataFrame. When \"legacy\",1. when Arrow optimization is disabled, convert to Row object, 2. when Arrow optimization is enabled, convert to dict or raise an Exception if there are duplicated nested field names. When \"row\", convert to Row object regardless of Arrow optimization. When \"dict\", convert to dict and use suffixed key names, e.g., a_0, a_1, if there are duplicated nested field names, regardless of Arrow optimization.",
+      "defaultValue": "legacy"
+    },
+    {
+      "key": "spark.sql.execution.pandas.udf.buffer.size",
+      "doc": "Same as `spark.buffer.size` but only applies to Pandas UDF executions. If it is not set, the fallback is `spark.buffer.size`. Note that Pandas execution requires more than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and pipelined; however, it might degrade performance. See SPARK-27870.",
+      "fallback": "spark.buffer.size"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.python",
+      "doc": "Python binary executable to use for PySpark in executors when running Python UDF, pandas UDF and pandas function APIs.If not set, it falls back to 'spark.pyspark.python' by default."
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled",
+      "doc": "When true, the traceback from Python UDFs is simplified. It hides the Python worker, (de)serialization, etc from PySpark in tracebacks, and only shows the exception messages from UDFs. Note that this works only with CPython 3.7+.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.arrow.enabled",
+      "doc": "Enable Arrow optimization in regular Python UDFs. This optimization can only be enabled when the given function takes at least one argument.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDTF.arrow.enabled",
+      "doc": "Enable Arrow optimization for Python UDTFs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.rangeExchange.sampleSizePerPartition",
+      "doc": "Number of points to sample per partition in order to determine the range boundaries for range partitioning, typically used in global sorting (without limit).",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.execution.removeRedundantProjects",
+      "doc": "Whether to remove redundant project exec node based on children's output and ordering requirement.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.removeRedundantSorts",
+      "doc": "Whether to remove redundant physical sort node",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.replaceHashWithSortAgg",
+      "doc": "Whether to replace hash aggregate node with sort aggregate based on children's ordering",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.reuseSubquery",
+      "doc": "When true, the planner will try to find out duplicated subqueries and re-use them.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.sortBeforeRepartition",
+      "doc": "When perform a repartition following a shuffle, the output row ordering would be nondeterministic. If some downstream stages fail and some tasks of the repartition stage retry, these tasks may generate different data, and that can lead to correctness issues. Turn on this config to insert a local sort before actually doing repartition to generate consistent repartition results. The performance of repartition() may go down since we insert extra local sort before it.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.topKSortFallbackThreshold",
+      "doc": "In SQL queries with a SORT followed by a LIMIT like 'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort in memory, otherwise do a global sort which spills to disk if necessary.",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.execution.useObjectHashAggregateExec",
+      "doc": "Decides if we use ObjectHashAggregateExec",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.usePartitionEvaluator",
+      "doc": "When true, use PartitionEvaluator to execute SQL operators.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.extensions",
+      "doc": "A comma-separated list of classes that implement Function1[SparkSessionExtensions, Unit] used to configure Spark Session extensions. The classes must have a no-args constructor. If multiple extensions are specified, they are applied in the specified order. For the case of rules and planner strategies, they are applied in the specified order. For the case of parsers, the last parser is used and each parser can delegate to its predecessor. For the case of function name conflicts, the last registered function name is used.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.files.ignoreCorruptFiles",
+      "doc": "Whether to ignore corrupt files. If true, the Spark jobs will continue to run when encountering corrupted files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.ignoreMissingFiles",
+      "doc": "Whether to ignore missing files. If true, the Spark jobs will continue to run when encountering missing files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.maxPartitionBytes",
+      "doc": "The maximum number of bytes to pack into a single partition when reading files. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "128MB"
+    },
+    {
+      "key": "spark.sql.files.maxPartitionNum",
+      "doc": "The suggested (not guaranteed) maximum number of split file partitions. If it is set, Spark will rescale each partition to make the number of partitions is close to this value if the initial number of partitions exceeds this value. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC."
+    },
+    {
+      "key": "spark.sql.files.maxRecordsPerFile",
+      "doc": "Maximum number of records to write out to a single file. If this value is zero or negative, there is no limit.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.files.minPartitionNum",
+      "doc": "The suggested (not guaranteed) minimum number of split file partitions. If not set, the default value is `spark.sql.leafNodeDefaultParallelism`. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC."
+    },
+    {
+      "key": "spark.sql.files.openCostInBytes",
+      "doc": "The estimated cost to open a file, measured by the number of bytes could be scanned in the same time. This is used when putting multiple files into a partition. It's better to over estimated, then the partitions with small files will be faster than partitions with bigger files (which is scheduled first). This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "4MB"
+    },
+    {
+      "key": "spark.sql.filesourceTableRelationCacheSize",
+      "doc": "The maximum size of the cache that maps qualified table names to table relation plans.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.fromJsonForceNullableSchema",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent errors like SPARK-23173 for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.function.concatBinaryAsString",
+      "doc": "When this option is set to false and all inputs are binary, `functions.concat` returns an output as binary. Otherwise, it returns as a string.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.function.eltOutputAsString",
+      "doc": "When this option is set to false and all inputs are binary, `elt` returns an output as binary. Otherwise, it returns as a string.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.globalTempDatabase",
+      "doc": "",
+      "defaultValue": "global_temp",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.groupByAliases",
+      "doc": "When true, aliases in a select list can be used in group by clauses. When false, an analysis exception is thrown in the case.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.groupByOrdinal",
+      "doc": "When true, the ordinal numbers in group by clauses are treated as the position in the select list. When false, the ordinal numbers are ignored.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.advancedPartitionPredicatePushdown.enabled",
+      "doc": "When true, advanced partition predicate pushdown into Hive metastore is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.caseSensitiveInferenceMode",
+      "doc": "Sets the action to take when a case-sensitive schema cannot be read from a Hive Serde table's properties when reading the table with Spark native data sources. Valid options include INFER_AND_SAVE (infer the case-sensitive schema from the underlying data files and write it back to the table properties), INFER_ONLY (infer the schema but don't attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback to using the case-insensitive metastore schema instead of inferring).",
+      "defaultValue": "NEVER_INFER"
+    },
+    {
+      "key": "spark.sql.hive.convertCTAS",
+      "doc": "When true, a table created by a Hive CTAS statement (no USING clause) without specifying any storage property will be converted to a data source table, using the data source set by spark.sql.sources.default.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.1",
+        "comment": "Set 'spark.sql.legacy.createHiveTableByDefault' to false instead."
+      }
+    },
+    {
+      "key": "spark.sql.hive.dropPartitionByName.enabled",
+      "doc": "When true, Spark will get partition name rather than partition object to drop partition, which can improve the performance of drop partition.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.filesourcePartitionFileCacheSize",
+      "doc": "When nonzero, enable caching of partition file metadata in memory. All tables share a cache that can use up to specified num bytes for file metadata. This conf only has an effect when hive filesource partition management is enabled.",
+      "defaultValue": "262144000"
+    },
+    {
+      "key": "spark.sql.hive.gatherFastStats",
+      "doc": "When true, fast stats (number of files and total size of all files) will be gathered in parallel while repairing table partitions to avoid the sequential listing in Hive metastore.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.manageFilesourcePartitions",
+      "doc": "When true, enable metastore partition management for file source tables as well. This includes both datasource and converted Hive tables. When partition management is enabled, datasource tables store partition in the Hive metastore, and use the metastore to prune partitions during query planning when spark.sql.hive.metastorePartitionPruning is set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruning",
+      "doc": "When true, some predicates will be pushed down into the Hive metastore so that unmatching partitions can be eliminated earlier.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningFallbackOnException",
+      "doc": "Whether to fallback to get all partitions from Hive metastore and perform partition pruning on Spark client side, when encountering MetaException from the metastore. Note that Spark query performance may degrade if this is enabled and there are many partitions to be listed. If this is disabled, Spark will fail the query instead.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningFastFallback",
+      "doc": "When this config is enabled, if the predicates are not supported by Hive or Spark does fallback due to encountering MetaException from the metastore, Spark will instead prune partitions by getting the partition names first and then evaluating the filter expressions on the client side. Note that the predicates with TimeZoneAwareExpression is not supported.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningInSetThreshold",
+      "doc": "The threshold of set size for InSet predicate when pruning partitions through Hive Metastore. When the set size exceeds the threshold, we rewrite the InSet predicate to be greater than or equal to the minimum value in set and less than or equal to the maximum value in set. Larger values may cause Hive Metastore stack overflow. But for InSet inside Not with values exceeding the threshold, we won't push it to Hive Metastore.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.hive.tablePropertyLengthThreshold",
+      "doc": "The maximum length allowed in a single cell when storing Spark-specific information in Hive's metastore as table properties. Currently it covers 2 things: the schema's JSON string, the histogram of column statistics."
+    },
+    {
+      "key": "spark.sql.hive.thriftServer.singleSession",
+      "doc": "When set to true, Hive Thrift server is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.hive.verifyPartitionPath",
+      "doc": "When true, check all the partition paths under the table's root directory when reading data stored in HDFS. This configuration will be deprecated in the future releases and replaced by spark.files.ignoreMissingFiles.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "This config is replaced by 'spark.files.ignoreMissingFiles'."
+      }
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.batchSize",
+      "doc": "Controls the size of batches for columnar caching.  Larger batch sizes can improve memory utilization and compression, but risk OOMs when caching data.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.compressed",
+      "doc": "When set to true Spark SQL will automatically select a compression codec for each column based on statistics of the data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.enableVectorizedReader",
+      "doc": "Enables vectorized reader for columnar caching.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.partitionPruning",
+      "doc": "When true, enable partition pruning for in-memory columnar tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryTableScanStatistics.enable",
+      "doc": "When true, enable in-memory table scan accumulators.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.join.preferSortMergeJoin",
+      "doc": "When true, prefer sort merge join over shuffled hash join. Sort merge join consumes less memory than shuffled hash join and it works efficiently when both join tables are large. On the other hand, shuffled hash join can improve performance (e.g., of full outer joins) when one of join tables is much smaller.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.enablePartialResults",
+      "doc": "When set to true, enables partial results for structs, maps, and arrays in JSON when one or more fields do not match the schema",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to JSON datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.jsonGenerator.ignoreNullFields",
+      "doc": "Whether to ignore null fields when generating JSON objects in JSON data source and JSON functions such as to_json. If false, it generates null for null fields in JSON objects.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.jsonGenerator.writeNullIfWithDefaultValue",
+      "doc": "When true, when writing NULL values to columns of JSON tables with explicit DEFAULT values using INSERT, UPDATE, or MERGE commands, never skip writing the NULL values to storage, overriding spark.sql.jsonGenerator.ignoreNullFields or the ignoreNullFields option. This can be useful to enforce that inserted NULL values are present in storage to differentiate from missing data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lateralColumnAlias.enableImplicitResolution",
+      "doc": "Enable resolving implicit lateral column alias defined in the same SELECT list. For example, with this conf turned on, for query `SELECT 1 AS a, a + 1` the `a` in `a + 1` can be resolved as the previously defined `1 AS a`. But note that table column has higher resolution priority than the lateral column alias.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.leafNodeDefaultParallelism",
+      "doc": "The default parallelism of Spark SQL leaf nodes that produce data, such as the file scan node, the local data scan node, the range node, etc. The default value of this config is 'SparkContext#defaultParallelism'."
+    },
+    {
+      "key": "spark.sql.legacy.addSingleFileInAddFile",
+      "doc": "When true, only a single file can be added using ADD FILE. If false, then users can add directory by passing directory path to ADD FILE.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowAutoGeneratedAliasForView",
+      "doc": "When true, it's allowed to use a input query without explicit alias when creating a permanent view.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent loss of user data for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.allowEmptySchemaWrite",
+      "doc": "When this option is set to true, validation of empty or empty nested schemas that occurs when writing into a FileFormat based data source does not happen.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowHashOnMapType",
+      "doc": "When set to true, hash expressions can be applied on elements of MapType. Otherwise, an analysis exception will be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNegativeScaleOfDecimal",
+      "doc": "When set to true, negative scale of Decimal type is allowed. For example, the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is Decimal(11, 0) in non legacy mode.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNonEmptyLocationInCTAS",
+      "doc": "When false, CTAS with LOCATION throws an analysis exception if the location is not empty.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNullComparisonResultInArraySort",
+      "doc": "When set to false, `array_sort` function throws an error if the comparator function returns null. If set to true, it restores the legacy behavior that handles null as zero (equal).",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowParameterlessCount",
+      "doc": "When true, the SQL function 'count' is allowed to take no parameters.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowStarWithSingleTableIdentifierInCount",
+      "doc": "When true, the SQL function 'count' is allowed to take single 'tblName.*' as parameter",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowTempViewCreationWithMultipleNameparts",
+      "doc": "When true, temp view creation Dataset APIs will allow the view creation even if the view name is multiple name parts. The extra name parts will be dropped during the view creation",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowUntypedScalaUDF",
+      "doc": "When set to true, user is allowed to use org.apache.spark.sql.functions.udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowZeroIndexInFormatString",
+      "doc": "When false, the `strfmt` in `format_string(strfmt, obj, ...)` and `printf(strfmt, obj, ...)` will no longer support to use \"0$\" to specify the first argument, the first argument should always reference by \"1$\" when use argument index to indicating the position of the argument in the argument list.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.avro.allowIncompatibleSchema",
+      "doc": "When set to false, if types in Avro are encoded in the same format, but the type in the Avro schema explicitly says that the data types are different, reject reading the data type in the format to avoid returning incorrect results. When set to true, it restores the legacy behavior of allow reading the data in the format, which may return incorrect results.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.bucketedTableScan.outputOrdering",
+      "doc": "When true, the bucketed table scan will list files during planning to figure out the output ordering, which is expensive and may make the planning quite slow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.castComplexTypesToString.enabled",
+      "doc": "When true, maps and structs are wrapped by [] in casting to strings, and NULL elements of structs/maps/arrays will be omitted while converting to strings. Otherwise, if this is false, which is the default, maps and structs are wrapped by {}, and NULL elements will be converted to \"null\".",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.charVarcharAsString",
+      "doc": "When true, Spark treats CHAR/VARCHAR type the same as STRING type, which is the behavior of Spark 3.0 and earlier. This means no length check for CHAR/VARCHAR type and no padding for CHAR type when writing data to the table.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.compareDateTimestampInTimestamp",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent errors like SPARK-23549 for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.createEmptyCollectionUsingStringType",
+      "doc": "When set to true, Spark returns an empty collection with `StringType` as element type if the `array`/`map` function is called without any parameters. Otherwise, Spark returns an empty collection with `NullType` as element type.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.createHiveTableByDefault",
+      "doc": "When set to true, CREATE TABLE syntax without USING or STORED AS will use Hive instead of the value of spark.sql.sources.default as the table provider.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.csv.enableDateTimeParsingFallback",
+      "doc": "When true, enable legacy date/time parsing fallback in CSV"
+    },
+    {
+      "key": "spark.sql.legacy.ctePrecedencePolicy",
+      "doc": "When LEGACY, outer CTE definitions takes precedence over inner definitions. If set to CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, AnalysisException is thrown while name conflict is detected in nested CTE. This config will be removed in future versions and CORRECTED will be the only behavior.",
+      "defaultValue": "EXCEPTION"
+    },
+    {
+      "key": "spark.sql.legacy.dataFrameWriterV2IgnorePathOption",
+      "doc": "When set to true, DataFrameWriterV2 ignores the 'path' option and always write data to the default table location.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue",
+      "doc": "When set to true, the key attribute resulted from running `Dataset.groupByKey` for non-struct key type, will be named as `value`, following the behavior of Spark version 2.4 and earlier.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.doLooseUpcast",
+      "doc": "When true, the upcast will be loose and allows string to atomic types.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.emptyCurrentDBInCli",
+      "doc": "When false, spark-sql CLI prints the the current database in prompt",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName",
+      "doc": "When true, columns will be looked up by name if labeled with a string and fallback to use position if not. When false, a grouped map Pandas UDF will assign columns from the returned Pandas DataFrame based on position, regardless of column label type. This configuration will be deprecated in future releases.",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "2.4",
+        "comment": "The config allows to switch to the behaviour before Spark 2.4 and will be removed in the future releases."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.exponentLiteralAsDecimal.enabled",
+      "doc": "When set to true, a literal with an exponent (e.g. 1E-30) would be parsed as Decimal rather than Double.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.extraOptionsBehavior.enabled",
+      "doc": "When true, the extra options will be ignored for DataFrameReader.table(). If set it to false, which is the default, Spark will check if the extra options have the same key, but the value is different with the table serde properties. If the check passes, the extra options will be merged with the serde properties as the scan options. Otherwise, an exception will be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.followThreeValuedLogicInArrayExists",
+      "doc": "When true, the ArrayExists will follow the three-valued boolean logic.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.fromDayTimeString.enabled",
+      "doc": "When true, the `from` bound is not taken into account in conversion of a day-time string to an interval, and the `to` bound is used to skip all interval units out of the specified range. If it is set to `false`, `ParseException` is thrown if the input does not match to the pattern defined by `from` and `to`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.groupingIdWithAppendedUserGroupBy",
+      "doc": "When true, grouping_id() returns values based on grouping set columns plus user-given group-by expressions order like Spark 3.2.0, 3.2.1, 3.2.2, and 3.3.0.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.histogramNumericPropagateInputType",
+      "doc": "The histogram_numeric function computes a histogram on numeric 'expr' using nb bins. The return value is an array of (x,y) pairs representing the centers of the histogram's bins. If this config is set to true, the output type of the 'x' field in the return value is propagated from the input value consumed in the aggregate function. Otherwise, 'x' always has double type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.inSubqueryNullability",
+      "doc": "When set to false, IN subquery nullability is correctly calculated based on both the left and right sides of the IN. When set to true, restores the legacy behavior that does not check the right side's nullability.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.integerGroupingId",
+      "doc": "When true, grouping_id() returns int values instead of long values.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.interval.enabled",
+      "doc": "When set to true, Spark SQL uses the mixed legacy interval type `CalendarIntervalType` instead of the ANSI compliant interval types `YearMonthIntervalType` and `DayTimeIntervalType`. For instance, the date subtraction expression returns `CalendarIntervalType` when the SQL config is set to `true` otherwise an ANSI interval.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.json.allowEmptyString.enabled",
+      "doc": "When set to true, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.json.enableDateTimeParsingFallback",
+      "doc": "When true, enable legacy date/time parsing fallback in JSON"
+    },
+    {
+      "key": "spark.sql.legacy.keepCommandOutputSchema",
+      "doc": "When true, Spark will keep the output schema of commands such as SHOW DATABASES unchanged.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.keepPartitionSpecAsStringLiteral",
+      "doc": "If it is set to true, `PARTITION(col=05)` is parsed as a string literal of its text representation, e.g., string '05', when the partition column is string type. Otherwise, it is always parsed as a numeric literal in the partition spec.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.literal.pickMinimumPrecision",
+      "doc": "When integral literal is used in decimal operations, pick a minimum precision required by the literal if this config is true, to make the resulting precision and/or scale smaller. This can reduce the possibility of precision lose and/or overflow.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.lpadRpadAlwaysReturnString",
+      "doc": "When set to false, when the first argument and the optional padding pattern is a byte sequence, the result is a BINARY value. The default padding pattern in this case is the zero byte. When set to true, it restores the legacy behavior of always returning string types even for binary inputs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mssqlserver.numericMapping.enabled",
+      "doc": "When true, use legacy MySqlServer SMALLINT and REAL type mapping.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.negativeIndexInArrayInsert",
+      "doc": "When set to true, restores the legacy behavior of `array_insert` for negative indexes - 0-based: the function inserts new element before the last one for the index -1. For example, `array_insert(['a', 'b'], -1, 'x')` returns `['a', 'x', 'b']`. When set to false, the -1 index points out to the last element, and the given example produces `['a', 'b', 'x']`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.noCharPaddingInPredicate",
+      "doc": "When true, Spark will not apply char type padding for CHAR type columns in string comparison predicates, when 'spark.sql.readSideCharPadding' is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.notReserveProperties",
+      "doc": "When true, all database and table properties are not reserved and available for create/alter syntaxes. But please be aware that the reserved properties will be silently removed.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.nullInEmptyListBehavior",
+      "doc": "When set to true, restores the legacy incorrect behavior of IN expressions for NULL values IN an empty list (including IN subqueries and literal IN lists): `null IN (empty list)` should evaluate to false, but sometimes (not always) incorrectly evaluates to null in the legacy behavior.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv",
+      "doc": "When set to false, nulls are written as unquoted empty strings in CSV data source. If set to true, it restores the legacy behavior that nulls were written as quoted empty strings, `\"\"`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parquet.nanosAsLong",
+      "doc": "When true, the Parquet's nanos precision timestamps are converted to SQL long values.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parseNullPartitionSpecAsStringLiteral",
+      "doc": "If it is set to true, `PARTITION(col=null)` is parsed as a string literal of its text representation, e.g., string 'null', when the partition column is string type. Otherwise, it is always parsed as a null literal in the partition spec.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parser.havingWithoutGroupByAsWhere",
+      "doc": "If it is set to true, the parser will treat HAVING without GROUP BY as a normal WHERE, which does not follow SQL standard.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.pathOptionBehavior.enabled",
+      "doc": "When true, \"path\" option is overwritten if one path parameter is passed to DataFrameReader.load(), DataFrameWriter.save(), DataStreamReader.load(), or DataStreamWriter.start(). Also, \"path\" option is added to the overall paths if multiple path parameters are passed to DataFrameReader.load()",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.percentileDiscCalculation",
+      "doc": "If true, the old bogus percentile_disc calculation is used. The old calculation incorrectly mapped the requested percentile to the sorted range of values in some cases and so returned incorrect results. Also, the new implementation is faster as it doesn't contain the interpolation logic that the old percentile_cont based one did.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.replaceDatabricksSparkAvro.enabled",
+      "doc": "If it is set to true, the data source provider com.databricks.spark.avro is mapped to the built-in but external Avro data source module for backward compatibility.",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use `.format(\"avro\")` in `DataFrameWriter` or `DataFrameReader` instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.respectNullabilityInTextDatasetConversion",
+      "doc": "When true, the nullability in the user-specified schema for `DataFrameReader.schema(schema).json(jsonDataset)` and `DataFrameReader.schema(schema).csv(csvDataset)` is respected. Otherwise, they are turned to a nullable schema forcibly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.sessionInitWithConfigDefaults",
+      "doc": "Flag to revert to legacy behavior where a cloned SparkSession receives SparkConf defaults, dropping any overrides in its parent SparkSession.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.legacy.setCommandRejectsSparkCoreConfs",
+      "doc": "If it is set to true, SET command will fail when the key is registered as a SparkConf entry.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.setopsPrecedence.enabled",
+      "doc": "When set to true and the order of evaluation is not specified by parentheses, the set operations are performed from left to right as they appear in the query. When set to false and order of evaluation is not specified by parentheses, INTERSECT operations are performed before any UNION, EXCEPT and MINUS operations.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.sizeOfNull",
+      "doc": "If it is set to false, or spark.sql.ansi.enabled is true, then size of null returns null. Otherwise, it returns -1, which was inherited from Hive.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.skipTypeValidationOnAlterPartition",
+      "doc": "When true, skip validation for partition spec in ALTER PARTITION. E.g., `ALTER TABLE .. ADD PARTITION(p='a')` would work even the partition type is int. When false, the behavior follows spark.sql.storeAssignmentPolicy",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.statisticalAggregate",
+      "doc": "When set to true, statistical aggregate function returns Double.NaN if divide by zero occurred during expression evaluation, otherwise, it returns null. Before version 3.1.0, it returns NaN in divideByZero case by default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.storeAnalyzedPlanForView",
+      "doc": "When true, analyzed plan instead of SQL text will be stored when creating temporary view",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.timeParserPolicy",
+      "doc": "When LEGACY, java.text.SimpleDateFormat is used for formatting and parsing dates/timestamps in a locale-sensitive manner, which is the approach before Spark 3.0. When set to CORRECTED, classes from java.time.* packages are used for the same purpose. The default value is EXCEPTION, RuntimeException is thrown when we will get different results.",
+      "defaultValue": "EXCEPTION"
+    },
+    {
+      "key": "spark.sql.legacy.typeCoercion.datetimeToString.enabled",
+      "doc": "If it is set to true, date/timestamp will cast to string in binary comparisons with String when spark.sql.ansi.enabled is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useCurrentConfigsForView",
+      "doc": "When true, SQL Configs of the current active SparkSession instead of the captured ones will be applied during the parsing and analysis phases of the view resolution.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useV1Command",
+      "doc": "When true, Spark will use legacy V1 SQL commands.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.v1IdentifierNoCatalog",
+      "doc": "When set to false, the v1 identifier will include 'spark_catalog' as the catalog name if database is defined. When set to true, it restores the legacy behavior that does not include catalog name.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.limit.initialNumPartitions",
+      "doc": "Initial number of partitions to try when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as morejobs will be run",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.limit.scaleUpFactor",
+      "doc": "Minimal increase rate in number of partitions between attempts when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as more jobs will be run",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.mapKeyDedupPolicy",
+      "doc": "The policy to deduplicate map keys in builtin function: CreateMap, MapFromArrays, MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted at last takes precedence.",
+      "defaultValue": "EXCEPTION"
+    },
+    {
+      "key": "spark.sql.maven.additionalRemoteRepositories",
+      "doc": "A comma-delimited string config of the optional additional remote Maven mirror repositories. This is only used for downloading Hive jars in IsolatedClientLoader if the default Maven Central repo is unreachable.",
+      "defaultValue": "https://maven-central.storage-download.googleapis.com/maven2/"
+    },
+    {
+      "key": "spark.sql.maxConcurrentOutputFileWriters",
+      "doc": "Maximum number of output file writers to use concurrently. If number of writers needed reaches this limit, task will sort rest of output then writing them.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.maxMetadataStringLength",
+      "doc": "Maximum number of characters to output for a metadata string. e.g. file location in `DataSourceScanExec`, every value will be abbreviated if exceed length.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.maxPlanStringLength",
+      "doc": "Maximum number of characters to output for a plan string.  If the plan is longer, further output will be truncated.  The default setting always generates a full plan.  Set this to a lower value such as 8k if plan strings are taking up too much memory or are causing OutOfMemory errors in the driver or UI processes.",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.maxSinglePartitionBytes",
+      "doc": "The maximum number of bytes allowed for a single partition. Otherwise, The planner will introduce shuffle to improve parallelism.",
+      "defaultValue": "9223372036854775807b"
+    },
+    {
+      "key": "spark.sql.metadataCacheTTLSeconds",
+      "doc": "Time-to-live (TTL) value for the metadata caches: partition file metadata cache and session catalog cache. This configuration only has an effect when this value having a positive value (> 0). It also requires setting 'spark.sql.catalogImplementation' to `hive`, setting 'spark.sql.hive.filesourcePartitionFileCacheSize' > 0 and setting 'spark.sql.hive.manageFilesourcePartitions' to `true` to be applied to the partition file metadata cache.",
+      "defaultValue": "-1000ms",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.objectHashAggregate.sortBased.fallbackThreshold",
+      "doc": "In the case of ObjectHashAggregateExec, when the size of the in-memory hash map grows too large, we will fall back to sort-based aggregation. This option sets a row count threshold for the size of the hash map.",
+      "defaultValue": "128"
+    },
+    {
+      "key": "spark.sql.optimizeNullAwareAntiJoin",
+      "doc": "When true, NULL-aware anti join execution will be planed into BroadcastHashJoinExec with flag isNullAwareAntiJoin enabled, optimized from O(M*N) calculation into O(M) calculation using Hash lookup instead of Looping lookup.Only support for singleColumn NAAJ for now.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.canChangeCachedPlanOutputPartitioning",
+      "doc": "Whether to forcibly enable some optimization rules that can change the output partitioning of a cached query when executing it for caching. If it is set to true, queries may need an extra shuffle to read the cached data. This configuration is enabled by default. The optimization rules enabled by this configuration are spark.sql.adaptive.enabled and spark.sql.sources.bucketing.autoBucketedScan.enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.collapseProjectAlwaysInline",
+      "doc": "Whether to always collapse two adjacent projections and inline expressions even if it causes extra duplication.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateInnerQuery.enabled",
+      "doc": "Decorrelate inner query by eliminating correlated references and build domain joins.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSetOps.enabled",
+      "doc": "Decorrelate subqueries with correlation under set operators.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSubqueryLegacyIncorrectCountHandling.enabled",
+      "doc": "If enabled, revert to legacy incorrect behavior for certain subqueries with COUNT or similar aggregates: see SPARK-43098.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.disableHints",
+      "doc": "When true, the optimizer will disable user-specified hints that are additional directives for better planning of a query.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.enabled",
+      "doc": "When true, we will generate predicate for partition column when it's used as join key",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio",
+      "doc": "When statistics are not available or configured not to be used, this config will be used as the fallback filter ratio for computing the data size of the partitioned table after dynamic partition pruning, in order to evaluate if it is worth adding an extra subquery as the pruning filter if broadcast reuse is not applicable.",
+      "defaultValue": "0.5"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly",
+      "doc": "When true, dynamic partition pruning will only apply when the broadcast exchange of a broadcast hash join operation can be reused as the dynamic pruning filter.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.useStats",
+      "doc": "When true, distinct count statistics will be used for computing the data size of the partitioned table after dynamic partition pruning, in order to evaluate if it is worth adding an extra subquery as the pruning filter if broadcast reuse is not applicable.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.enableCsvExpressionOptimization",
+      "doc": "Whether to optimize CSV expressions in SQL optimizer. It includes pruning unnecessary columns from from_csv.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.enableJsonExpressionOptimization",
+      "doc": "Whether to optimize JSON expressions in SQL optimizer. It includes pruning unnecessary columns from from_json, simplifying from_json + to_json, to_json + named_struct(from_json.col1, from_json.col2, ....).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.excludeSubqueryRefsFromRemoveRedundantAliases.enabled",
+      "doc": "When true, exclude the references from the subquery expressions (in, exists, etc.) while removing redundant aliases.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.excludedRules",
+      "doc": "Configures a list of rules to be disabled in the optimizer, in which the rules are specified by their rule names and separated by comma. It is not guaranteed that all the rules in this configuration will eventually be excluded, as some rules are necessary for correctness. The optimizer will log the rules that have indeed been excluded."
+    },
+    {
+      "key": "spark.sql.optimizer.expression.nestedPruning.enabled",
+      "doc": "Prune nested fields from expressions in an operator which are unnecessary in satisfying a query. Note that this optimization doesn't prune nested fields from physical data source scanning. For pruning nested fields from scanning, please use `spark.sql.optimizer.nestedSchemaPruning.enabled` config.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.expressionProjectionCandidateLimit",
+      "doc": "The maximum number of the candidate of output expressions whose alias are replaced. It can preserve the output partitioning and ordering. Negative value means disable this optimization.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.optimizer.inSetConversionThreshold",
+      "doc": "The threshold of set size for InSet conversion.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.optimizer.inSetSwitchThreshold",
+      "doc": "Configures the max set size in InSet for which Spark will generate code with switch statements. This is applicable only to bytes, shorts, ints, dates.",
+      "defaultValue": "400"
+    },
+    {
+      "key": "spark.sql.optimizer.maxIterations",
+      "doc": "The max number of iterations the optimizer runs.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.optimizer.metadataOnly",
+      "doc": "When true, enable the metadata-only query optimization that use the table's metadata to produce the partition columns instead of table scans. It applies when all the columns scanned are partition columns and the query has an aggregate operator that satisfies distinct semantics. By default the optimization is disabled, and deprecated as of Spark 3.0 since it may return incorrect results when the files are empty, see also SPARK-26709.It will be removed in the future releases. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Avoid to depend on this optimization to prevent a potential correctness issue. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources",
+      "doc": "A comma-separated list of data source short names or fully qualified data source implementation class names for which Spark tries to push down predicates for nested columns and/or names containing `dots` to data sources. This configuration is only effective with file-based data sources in DSv1. Currently, Parquet and ORC implement both optimizations. The other data sources don't support this feature yet. So the default value is 'parquet,orc'.",
+      "defaultValue": "parquet,orc"
+    },
+    {
+      "key": "spark.sql.optimizer.nestedSchemaPruning.enabled",
+      "doc": "Prune nested fields from a logical relation's output which are unnecessary in satisfying a query. This optimization allows columnar file format readers to avoid reading unnecessary nested column data. Currently Parquet and ORC are the data sources that implement this optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery",
+      "doc": "When true, the optimizer will inline subqueries with OneRowRelation as leaf nodes.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline",
+      "doc": "When true, the optimizer will always inline single row subqueries even if it causes extra duplication. It only takes effect when spark.sql.optimizer.optimizeOneRowRelationSubquery is set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.batches",
+      "doc": "(Removed)",
+      "defaultValue": "",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.batches` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.level",
+      "doc": "(Removed)",
+      "defaultValue": "trace",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.level` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.rules",
+      "doc": "(Removed)",
+      "defaultValue": "",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.rules` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.plannedWrite.enabled",
+      "doc": "When set to true, Spark optimizer will add logical sort operators to V1 write commands if needed so that `FileFormatWriter` does not need to insert physical sorts.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.propagateDistinctKeys.enabled",
+      "doc": "When true, the query optimizer will propagate a set of distinct attributes from the current node and use it to optimize query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.pruneFiltersCanPruneStreamingSubplan",
+      "doc": "Allow PruneFilters to remove streaming subplans when we encounter a false filter. This flag is to restore prior buggy behavior for broken pipelines.",
+      "defaultValue": "false",
+      "alternatives": [
+        "spark.databricks.sql.optimizer.pruneFiltersCanPruneStreamingSubplan"
+      ]
+    },
+    {
+      "key": "spark.sql.optimizer.pullHintsIntoSubqueries",
+      "doc": "Pull hints into subqueries in EliminateResolvedHint if enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.replaceExceptWithFilter",
+      "doc": "When true, the apply function of the rule verifies whether the right node of the except operation is of type Filter or Project followed by Filter. If yes, the rule further verifies 1) Excluding the filter operations from the right (as well as the left node, if any) on the top, whether both the nodes evaluates to a same result. 2) The left and right nodes don't contain any SubqueryExpressions. 3) The output column names of the left node are distinct. If all the conditions are met, the rule will replace the except operation with a Filter by flipping the filter condition(s) of the right node.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold",
+      "doc": "Byte size threshold of the Bloom filter application side plan's aggregated scan size. Aggregated scan byte size of the Bloom filter application side needs to be over this value to inject a bloom filter.",
+      "defaultValue": "10GB"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold",
+      "doc": "Size threshold of the bloom filter creation side plan. Estimated size needs to be under this value to try to inject bloom filter.",
+      "defaultValue": "10MB"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.enabled",
+      "doc": "When true and if one side of a shuffle join has a selective predicate, we attempt to insert a bloom filter in the other side to reduce the amount of shuffle data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.expectedNumItems",
+      "doc": "The default number of expected items for the runtime bloomfilter",
+      "defaultValue": "1000000"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.maxNumBits",
+      "doc": "The max number of bits to use for the runtime bloom filter",
+      "defaultValue": "67108864"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.maxNumItems",
+      "doc": "The max allowed number of expected items for the runtime bloom filter",
+      "defaultValue": "4000000"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.numBits",
+      "doc": "The default number of bits to use for the runtime bloom filter",
+      "defaultValue": "8388608"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.rowLevelOperationGroupFilter.enabled",
+      "doc": "Enables runtime group filtering for group-based row-level operations. Data sources that replace groups of data (e.g. files, partitions) may prune entire groups using provided data source filters when planning a row-level operation scan. However, such filtering is limited as not all expressions can be converted into data source filters and some expressions can only be evaluated by Spark (e.g. subqueries). Since rewriting groups is expensive, Spark can execute a query at runtime to find what records match the condition of the row-level operation. The information about matching records will be passed back to the row-level operation scan, allowing data sources to discard groups that don't have to be rewritten.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtimeFilter.number.threshold",
+      "doc": "The total number of injected runtime filters (non-DPP) for a single query. This is to prevent driver OOMs with too many Bloom filters.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.optimizer.runtimeFilter.semiJoinReduction.enabled",
+      "doc": "When true and if one side of a shuffle join has a selective predicate, we attempt to insert a semi join in the other side to reduce the amount of shuffle data.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.serializer.nestedSchemaPruning.enabled",
+      "doc": "Prune nested fields from object serialization operator which are unnecessary in satisfying a query. This optimization allows object serializers to avoid executing unnecessary nested expressions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.windowGroupLimitThreshold",
+      "doc": "Threshold for triggering `InsertWindowGroupLimit`. 0 means the output results is empty. -1 means disabling the optimization.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.orc.aggregatePushdown",
+      "doc": "If true, aggregates will be pushed down to ORC for optimization. Support MIN, MAX and COUNT as aggregate expression. For MIN/MAX, support boolean, integer, float and date type. For COUNT, support all data types. If statistics is missing from any ORC file footer, exception would be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.orc.columnarReaderBatchSize",
+      "doc": "The number of rows to include in a orc vectorized reader batch. The number should be carefully chosen to minimize overhead and avoid OOMs in reading data.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.orc.columnarWriterBatchSize",
+      "doc": "The number of rows to include in a orc vectorized writer batch. The number should be carefully chosen to minimize overhead and avoid OOMs in writing data.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.orc.compression.codec",
+      "doc": "Sets the compression codec used when writing ORC files. If either `compression` or `orc.compress` is specified in the table-specific options/properties, the precedence would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`.Acceptable values include: none, uncompressed, snappy, zlib, lzo, zstd, lz4.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.orc.enableNestedColumnVectorizedReader",
+      "doc": "Enables vectorized orc decoding for nested column.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.enableVectorizedReader",
+      "doc": "Enables vectorized orc decoding.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.filterPushdown",
+      "doc": "When true, enable filter pushdown for ORC files.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.impl",
+      "doc": "When native, use the native version of ORC support instead of the ORC library in Hive. It is 'hive' by default prior to Spark 2.4.",
+      "defaultValue": "native"
+    },
+    {
+      "key": "spark.sql.orc.mergeSchema",
+      "doc": "When true, the Orc data source merges schemas collected from all data files, otherwise the schema is picked from a random data file.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.orderByOrdinal",
+      "doc": "When true, the ordinal numbers are treated as the position in the select list. When false, the ordinal numbers in order/sort by clause are ignored.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.aggregatePushdown",
+      "doc": "If true, aggregates will be pushed down to Parquet for optimization. Support MIN, MAX and COUNT as aggregate expression. For MIN/MAX, support boolean, integer, float and date type. For COUNT, support all data types. If statistics is missing from any Parquet file footer, exception would be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.binaryAsString",
+      "doc": "Some other Parquet-producing systems, in particular Impala and older versions of Spark SQL, do not differentiate between binary data and strings when writing out the Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.columnarReaderBatchSize",
+      "doc": "The number of rows to include in a parquet vectorized reader batch. The number should be carefully chosen to minimize overhead and avoid OOMs in reading data.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.parquet.compression.codec",
+      "doc": "Sets the compression codec used when writing Parquet files. If either `compression` or `parquet.compression` is specified in the table-specific options/properties, the precedence would be `compression`, `parquet.compression`, `spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, snappy, gzip, lzo, brotli, lz4, lz4raw, lz4_raw, zstd.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.parquet.datetimeRebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. When EXCEPTION, which is the default, Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown. This config influences on reads of the following parquet logical types: DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: spark.sql.parquet.int96RebaseModeInRead.",
+      "defaultValue": "EXCEPTION",
+      "alternatives": [
+        "spark.sql.legacy.parquet.datetimeRebaseModeInRead"
+      ]
+    },
+    {
+      "key": "spark.sql.parquet.datetimeRebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. When EXCEPTION, which is the default, Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config influences on writes of the following parquet logical types: DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: spark.sql.parquet.int96RebaseModeInWrite.",
+      "defaultValue": "EXCEPTION",
+      "alternatives": [
+        "spark.sql.legacy.parquet.datetimeRebaseModeInWrite"
+      ]
+    },
+    {
+      "key": "spark.sql.parquet.enableNestedColumnVectorizedReader",
+      "doc": "Enables vectorized Parquet decoding for nested columns (e.g., struct, list, map). Requires spark.sql.parquet.enableVectorizedReader to be enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.enableVectorizedReader",
+      "doc": "Enables vectorized parquet decoding.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.read.enabled",
+      "doc": "Field ID is a native field of the Parquet schema spec. When enabled, Parquet readers will use field IDs (if present) in the requested Spark schema to look up Parquet fields instead of using column names",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.read.ignoreMissing",
+      "doc": "When the Parquet file doesn't have any field IDs but the Spark read schema is using field IDs to read, we will silently return nulls when this flag is enabled, or error otherwise.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.write.enabled",
+      "doc": "Field ID is a native field of the Parquet schema spec. When enabled, Parquet writers will populate the field Id metadata (if present) in the Spark schema to the Parquet schema.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown",
+      "doc": "Enables Parquet filter push-down optimization when set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.date",
+      "doc": "If true, enables Parquet filter push-down optimization for Date. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.decimal",
+      "doc": "If true, enables Parquet filter push-down optimization for Decimal. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.string.startsWith",
+      "doc": "If true, enables Parquet filter push-down optimization for string startsWith function. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.stringPredicate",
+      "doc": "If true, enables Parquet filter push-down optimization for string predicate such as startsWith/endsWith/contains function. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "fallback": "spark.sql.parquet.filterPushdown.string.startsWith"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.timestamp",
+      "doc": "If true, enables Parquet filter push-down optimization for Timestamp. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.inferTimestampNTZ.enabled",
+      "doc": "When enabled, Parquet timestamp columns with annotation isAdjustedToUTC = false are inferred as TIMESTAMP_NTZ type during schema inference. Otherwise, all the Parquet timestamp columns are inferred as TIMESTAMP_LTZ types. Note that Spark writes the output schema into Parquet's footer metadata on file writing and leverages it on file reading. Thus this configuration only affects the schema inference on Parquet files which are not written by Spark.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.int64AsTimestampMillis",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "The config was deprecated since Spark 2.3.Use 'spark.sql.parquet.outputTimestampType' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.parquet.int96AsTimestamp",
+      "doc": "Some Parquet-producing systems, in particular Impala, store Timestamp into INT96. Spark would also store Timestamp as INT96 because we need to avoid precision lost of the nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.int96RebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase INT96 timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. When CORRECTED, Spark will not do rebase and read the timestamps as it is. When EXCEPTION, which is the default, Spark will fail the reading if it sees ancient timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.",
+      "defaultValue": "EXCEPTION",
+      "alternatives": [
+        "spark.sql.legacy.parquet.int96RebaseModeInRead"
+      ]
+    },
+    {
+      "key": "spark.sql.parquet.int96RebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase INT96 timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. When CORRECTED, Spark will not do rebase and write the timestamps as it is. When EXCEPTION, which is the default, Spark will fail the writing if it sees ancient timestamps that are ambiguous between the two calendars.",
+      "defaultValue": "EXCEPTION",
+      "alternatives": [
+        "spark.sql.legacy.parquet.int96RebaseModeInWrite"
+      ]
+    },
+    {
+      "key": "spark.sql.parquet.int96TimestampConversion",
+      "doc": "This controls whether timestamp adjustments should be applied to INT96 data when converting to timestamps, for data written by Impala.  This is necessary because Impala stores INT96 data with a different timezone offset than Hive & Spark.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.mergeSchema",
+      "doc": "When true, the Parquet data source merges schemas collected from all data files, otherwise the schema is picked from the summary file or a random data file if no summary file is available.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.output.committer.class",
+      "doc": "The output committer class used by Parquet. The specified class needs to be a subclass of org.apache.hadoop.mapreduce.OutputCommitter. Typically, it's also a subclass of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata summaries will never be created, irrespective of the value of parquet.summary.metadata.level",
+      "defaultValue": "org.apache.parquet.hadoop.ParquetOutputCommitter"
+    },
+    {
+      "key": "spark.sql.parquet.outputTimestampType",
+      "doc": "Sets which Parquet timestamp type to use when Spark writes data to Parquet files. INT96 is a non-standard but commonly used timestamp type in Parquet. TIMESTAMP_MICROS is a standard timestamp type in Parquet, which stores number of microseconds from the Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which means Spark has to truncate the microsecond portion of its timestamp value.",
+      "defaultValue": "INT96"
+    },
+    {
+      "key": "spark.sql.parquet.pushdown.inFilterThreshold",
+      "doc": "For IN predicate, Parquet filter will push-down a set of OR clauses if its number of values not exceeds this threshold. Otherwise, Parquet filter will push-down a value greater than or equal to its minimum value and less than or equal to its maximum value. By setting this value to 0 this feature can be disabled. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.parquet.recordLevelFilter.enabled",
+      "doc": "If true, enables Parquet's native record-level filtering using the pushed down filters. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and the vectorized reader is not used. You can ensure the vectorized reader is not used by setting 'spark.sql.parquet.enableVectorizedReader' to false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.respectSummaryFiles",
+      "doc": "When true, we make assumption that all part-files of Parquet are consistent with summary files and we will ignore them when merging schema. Otherwise, if this is false, which is the default, we will merge all part-files. This should be considered as expert-only option, and shouldn't be enabled before knowing what it means exactly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.writeLegacyFormat",
+      "doc": "If true, data will be written in a way of Spark 1.4 and earlier. For example, decimal values will be written in Apache Parquet's fixed-length byte array format, which other systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet will be used. For example, decimals will be written in int-based format. If Parquet output is intended for use with systems that do not support this newer format, set to true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parser.escapedStringLiterals",
+      "doc": "When true, string literals (including regex patterns) remain escaped in our SQL parser. The default is false since Spark 2.0. Setting it to true can restore the behavior prior to Spark 2.0.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parser.quotedRegexColumnNames",
+      "doc": "When true, quoted Identifiers (using backticks) in SELECT statement are interpreted as regular expressions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pivotMaxValues",
+      "doc": "When doing a pivot without specifying values for the pivot column this is the maximum number of (distinct) values that will be collected without error.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.planChangeLog.batches",
+      "doc": "Configures a list of batches for logging plan changes, in which the batches are specified by their batch names and separated by comma."
+    },
+    {
+      "key": "spark.sql.planChangeLog.level",
+      "doc": "Configures the log level for logging the change from the original plan to the new plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'trace'.",
+      "defaultValue": "trace"
+    },
+    {
+      "key": "spark.sql.planChangeLog.rules",
+      "doc": "Configures a list of rules for logging plan changes, in which the rules are specified by their rule names and separated by comma."
+    },
+    {
+      "key": "spark.sql.planChangeValidation",
+      "doc": "If true, Spark will validate all the plan changes made by analyzer/optimizer and other catalyst rules, to make sure every rule returns a valid plan",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.inferNestedDictAsStruct.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the nested dict as a map by default. When it set to true, it infers the nested dict as a struct.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.jvmStacktrace.enabled",
+      "doc": "When true, it shows the JVM stacktrace in the user-facing PySpark exception together with Python stacktrace. By default, it is disabled to hide JVM stacktrace and shows a Python-friendly exception only. Note that this is independent from log level settings.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the element type of an array from all values in the array by default. If this config is set to true, it restores the legacy behavior of only inferring the type from the first array element.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.queryExecutionListeners",
+      "doc": "List of class names implementing QueryExecutionListener that will be automatically added to newly created sessions. The classes should have either a no-arg constructor, or a constructor that expects a SparkConf argument.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.readSideCharPadding",
+      "doc": "When true, Spark applies string padding when reading CHAR type columns/fields, in addition to the write-side padding. This config is true by default to better enforce CHAR type semantic in cases such as external tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.redaction.options.regex",
+      "doc": "Regex to decide which keys in a Spark SQL command's options map contain sensitive information. The values of options whose names that match this regex will be redacted in the explain output. This redaction is applied on top of the global redaction configuration defined by spark.redaction.regex.",
+      "defaultValue": "(?i)url"
+    },
+    {
+      "key": "spark.sql.redaction.string.regex",
+      "doc": "Regex to decide which parts of strings produced by Spark contain sensitive information. When this regex matches a string part, that string part is replaced by a dummy value. This is currently used to redact the output of SQL explain commands. When this conf is not set, the value from `spark.redaction.string.regex` is used.",
+      "fallback": "spark.redaction.string.regex"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.enabled",
+      "doc": "Enables eager evaluation or not. When true, the top K rows of Dataset will be displayed if and only if the REPL supports the eager evaluation. Currently, the eager evaluation is supported in PySpark and SparkR. In PySpark, for the notebooks like Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned outputs are showed similar to R data.frame would.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.maxNumRows",
+      "doc": "The max number of rows that are returned by eager evaluation. This only takes effect when spark.sql.repl.eagerEval.enabled is set to true. The valid range of this config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).",
+      "defaultValue": "20"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.truncate",
+      "doc": "The max number of characters for each cell that is returned by eager evaluation. This only takes effect when spark.sql.repl.eagerEval.enabled is set to true.",
+      "defaultValue": "20"
+    },
+    {
+      "key": "spark.sql.requireAllClusterKeysForCoPartition",
+      "doc": "When true, the planner requires all the clustering keys as the hash partition keys of the children, to eliminate the shuffles for the operator that needs its children to be co-partitioned, such as JOIN node. This is to avoid data skews which can lead to significant performance regression if shuffles are eliminated.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.requireAllClusterKeysForDistribution",
+      "doc": "When true, the planner requires all the clustering keys as the partition keys (with same ordering) of the children, to eliminate the shuffle for the operator that requires its children be clustered distributed, such as AGGREGATE and WINDOW node. This is to avoid data skews which can lead to significant performance regression if shuffle is eliminated.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.retainGroupColumns",
+      "doc": "",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.runSQLOnFiles",
+      "doc": "When true, we could use `datasource`.`path` as table in SQL query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.scriptTransformation.exitTimeoutInSeconds",
+      "doc": "Timeout for executor to wait for the termination of transformation script when EOF.",
+      "defaultValue": "10000ms"
+    },
+    {
+      "key": "spark.sql.selfJoinAutoResolveAmbiguity",
+      "doc": "",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.session.localRelationCacheThreshold",
+      "doc": "The threshold for the size in bytes of local relations to be cached at the driver side after serialization.",
+      "defaultValue": "67108864"
+    },
+    {
+      "key": "spark.sql.session.timeZone",
+      "doc": "The ID of session local timezone in the format of either region-based zone IDs or zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format '(+|-)HH', '(+|-)HH:mm' or '(+|-)HH:mm:ss', e.g '-08', '+01:00' or '-13:33:33'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous.",
+      "defaultValue": null
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.in.memory.threshold",
+      "doc": "Threshold for number of windows guaranteed to be held in memory by the session window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.shuffle.partitions",
+      "doc": "The default number of partitions to use when shuffling data for joins or aggregations. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.shuffledHashJoinFactor",
+      "doc": "The shuffle hash join can be selected if the data size of small side multiplied by this factor is still smaller than the large side.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.sort.enableRadixSort",
+      "doc": "When true, enable use of radix sort when possible. Radix sort is much faster but requires additional memory to be reserved up-front. The memory overhead may be significant when sorting very small rows (up to 50% more in this case).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the sort merge join operator",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by sort merge join operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.sources.binaryFile.maxLength",
+      "doc": "The max length of a file that can be read by the binary file data source. Spark will fail fast and not attempt to read the file if its length exceeds this value. The theoretical max is Int.MaxValue, though VMs might implement a smaller max.",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.autoBucketedScan.enabled",
+      "doc": "When true, decide whether to do bucketed scan on input tables based on query plan automatically. Do not use bucketed scan if 1. query does not have operators to utilize bucketing (e.g. join, group-by, etc), or 2. there's an exchange operator between these operators and table scan. Note when 'spark.sql.sources.bucketing.enabled' is set to false, this configuration does not take any effect.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.enabled",
+      "doc": "When false, we will treat bucketed table as normal table",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.maxBuckets",
+      "doc": "The maximum number of buckets allowed.",
+      "defaultValue": "100000"
+    },
+    {
+      "key": "spark.sql.sources.commitProtocolClass",
+      "doc": "",
+      "defaultValue": "org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol"
+    },
+    {
+      "key": "spark.sql.sources.default",
+      "doc": "The default data source to use in input/output.",
+      "defaultValue": "parquet"
+    },
+    {
+      "key": "spark.sql.sources.disabledJdbcConnProviderList",
+      "doc": "Configures a list of JDBC connection providers, which are disabled. The list contains the name of the JDBC connection providers separated by comma.",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.sources.fileCompressionFactor",
+      "doc": "When estimating the output data size of a table scan, multiply the file size with this factor as the estimated data size, in case the data is compressed in the file and lead to a heavily underestimated result.",
+      "defaultValue": "1.0"
+    },
+    {
+      "key": "spark.sql.sources.ignoreDataLocality",
+      "doc": "If true, Spark will not fetch the block locations for each file on listing files. This speeds up file listing, but the scheduler cannot schedule tasks to take advantage of data locality. It can be particularly useful if data is read from a remote cluster so the scheduler could never take advantage of locality anyway.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.outputCommitterClass",
+      "doc": ""
+    },
+    {
+      "key": "spark.sql.sources.parallelPartitionDiscovery.parallelism",
+      "doc": "The number of parallelism to list a collection of path recursively, Set the number to prevent file listing from generating too many tasks.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.sources.parallelPartitionDiscovery.threshold",
+      "doc": "The maximum number of paths allowed for listing files at driver side. If the number of detected paths exceeds this value during partition discovery, it tries to list the files with another Spark distributed job. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "32"
+    },
+    {
+      "key": "spark.sql.sources.partitionColumnTypeInference.enabled",
+      "doc": "When true, automatically infer the data types for partitioned columns.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.partitionOverwriteMode",
+      "doc": "When INSERT OVERWRITE a partitioned data source table, we currently support 2 modes: static and dynamic. In static mode, Spark deletes all the partitions that match the partition specification(e.g. PARTITION(a=1,b)) in the INSERT statement, before overwriting. In dynamic mode, Spark doesn't delete partitions ahead, and only overwrite those partitions that have data written into it at runtime. By default we use static mode to keep the same behavior of Spark prior to 2.3. Note that this config doesn't affect Hive serde tables, as they are always overwritten with dynamic mode. This can also be set as an output option for a data source using key partitionOverwriteMode (which takes precedence over this setting), e.g. dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path).",
+      "defaultValue": "STATIC"
+    },
+    {
+      "key": "spark.sql.sources.schemaStringLengthThreshold",
+      "doc": "The maximum length allowed in a single cell when storing additional schema information in Hive's metastore.",
+      "defaultValue": "4000",
+      "isStatic": true,
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use 'spark.sql.hive.tablePropertyLengthThreshold' instead."
+      }
+    },
+    {
+      "key": "spark.sql.sources.useV1SourceList",
+      "doc": "A comma-separated list of data source short names or fully qualified data source implementation class names for which Data Source V2 code path is disabled. These data sources will fallback to Data Source V1 code path.",
+      "defaultValue": "avro,csv,json,kafka,orc,parquet,text"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.enabled",
+      "doc": "Similar to spark.sql.sources.bucketing.enabled, this config is used to enable bucketing for V2 data sources. When turned on, Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and will try to avoid shuffle if necessary.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled",
+      "doc": "During a storage-partitioned join, whether to allow input partitions to be partially clustered, when both sides of the join are of KeyGroupedPartitioning. At planning time, Spark will pick the side with less data size based on table statistics, group and replicate them to match the other side. This is an optimization on skew join and can help to reduce data skewness when certain partitions are assigned large amount of data. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.pushPartValues.enabled",
+      "doc": "Whether to pushdown common partition values when spark.sql.sources.v2.bucketing.enabled is enabled. When turned on, if both sides of a join are of KeyGroupedPartitioning and if they share compatible partition keys, even if they don't have the exact same partition values, Spark will calculate a superset of partition values and pushdown that info to scan nodes, which will use empty partitions for the missing partition values on either side. This could help to eliminate unnecessary shuffles",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.validatePartitionColumns",
+      "doc": "When this option is set to true, partition column values will be validated with user-specified schema. If the validation fails, a runtime exception is thrown. When this option is set to false, the partition column value will be converted to null if it can not be casted to corresponding user-specified schema.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.stableDerivedColumnAlias.enabled",
+      "doc": "Enable deriving of stable column aliases from the lexer tree instead of parse tree and form them via pretty SQL print.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.fallBackToHdfs",
+      "doc": "When true, it will fall back to HDFS if the table statistics are not available from table metadata. This is useful in determining if a table is small enough to use broadcast joins. This flag is effective only for non-partitioned Hive tables. For non-partitioned data source tables, it will be automatically recalculated if table statistics are not available. For partitioned data source and partitioned Hive tables, It is 'spark.sql.defaultSizeInBytes' if table statistics are not available.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.histogram.enabled",
+      "doc": "Generates histograms when computing column statistics if enabled. Histograms can provide better estimation accuracy. Currently, Spark only supports equi-height histogram. Note that collecting histograms takes extra cost. For example, collecting column statistics usually takes only one table scan, but generating equi-height histogram will cause an extra table scan.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.histogram.numBins",
+      "doc": "The number of bins when generating histograms.",
+      "defaultValue": "254"
+    },
+    {
+      "key": "spark.sql.statistics.ndv.maxError",
+      "doc": "The maximum relative standard deviation allowed in HyperLogLog++ algorithm when generating column level statistics.",
+      "defaultValue": "0.05"
+    },
+    {
+      "key": "spark.sql.statistics.parallelFileListingInStatsComputation.enabled",
+      "doc": "When true, SQL commands use parallel file listing, as opposed to single thread listing. This usually speeds up commands that need to list many directories.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.statistics.percentile.accuracy",
+      "doc": "Accuracy of percentile approximation when generating equi-height histograms. Larger value means better accuracy. The relative error can be deduced by 1.0 / PERCENTILE_ACCURACY.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.statistics.size.autoUpdate.enabled",
+      "doc": "Enables automatic update for table size once table's data is changed. Note that if the total number of files of the table is very large, this can be expensive and slow down data change commands.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.storeAssignmentPolicy",
+      "doc": "When inserting a value into a column with different data type, Spark will perform type coercion. Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy, Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL. It disallows certain unreasonable type conversions such as converting `string` to `int` or `double` to `boolean`. With legacy policy, Spark allows the type coercion as long as it is a valid `Cast`, which is very loose. e.g. converting `string` to `int` or `double` to `boolean` is allowed. It is also the only behavior in Spark 2.x and it is compatible with Hive. With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is not allowed.",
+      "defaultValue": "ANSI"
+    },
+    {
+      "key": "spark.sql.streaming.aggregation.stateFormatVersion",
+      "doc": "State format version used by streaming aggregation operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.asyncLogPurge.enabled",
+      "doc": "When true, purging the offset log and commit log of old entries will be done asynchronously.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.escapedPathCheck.enabled",
+      "doc": "Whether to detect a streaming query may pick up an incorrect checkpoint path due to SPARK-26824.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.renamedFileCheck.enabled",
+      "doc": "When true, Spark will validate if renamed checkpoint file exists.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.checkpointLocation",
+      "doc": "The default location for storing checkpoint data for streaming queries."
+    },
+    {
+      "key": "spark.sql.streaming.commitProtocolClass",
+      "doc": "",
+      "defaultValue": "org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.epochBacklogQueueSize",
+      "doc": "The max number of entries to be stored in queue to wait for late epochs. If this parameter is exceeded by the size of the queue, stream will stop with an error.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.executorPollIntervalMs",
+      "doc": "The interval at which continuous execution readers will poll to check whether the epoch has advanced on the driver.",
+      "defaultValue": "100ms"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.executorQueueSize",
+      "doc": "The size (measured in number of rows) of the queue used in continuous execution to buffer the results of a ContinuousDataReader.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.streaming.disabledV2MicroBatchReaders",
+      "doc": "A comma-separated list of fully qualified data source register class names for which MicroBatchReadSupport is disabled. Reads from these sources will fall back to the V1 Sources.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.streaming.disabledV2Writers",
+      "doc": "A comma-separated list of fully qualified data source register class names for which StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.cleanupDelay",
+      "doc": "How long that a file is guaranteed to be visible for all readers.",
+      "defaultValue": "600000ms"
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.compactInterval",
+      "doc": "Number of log files after which all the previous files are compacted into the next log file.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.deletion",
+      "doc": "Whether to delete the expired log files in file stream sink.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.cleaner.numThreads",
+      "doc": "Number of threads used in the file source completed file cleaner.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.cleanupDelay",
+      "doc": "How long in milliseconds a file is guaranteed to be visible for all readers.",
+      "defaultValue": "600000ms"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.compactInterval",
+      "doc": "Number of log files after which all the previous files are compacted into the next log file.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.deletion",
+      "doc": "Whether to delete the expired log files in file stream source.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.schema.forceNullable",
+      "doc": "When true, force the schema of streaming file source to be nullable (including all the fields). Otherwise, the schema might not be compatible with actual data, which leads to corruptions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileStreamSink.ignoreMetadata",
+      "doc": "If this is enabled, when Spark reads from the results of a streaming query written by `FileStreamSink`, Spark will ignore the metadata log and treat it as normal path to read, e.g. listing files using HDFS APIs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion",
+      "doc": "State format version used by flatMapGroupsWithState operation in a streaming query",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.forceDeleteTempCheckpointLocation",
+      "doc": "When true, enable temporary checkpoint locations force delete.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.join.stateFormatVersion",
+      "doc": "State format version used by streaming join operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.kafka.useDeprecatedOffsetFetching",
+      "doc": "When true, the deprecated Consumer based offset fetching used which could cause infinite wait in Spark queries. Such cases query restart is the only workaround. For further details please see Offset Fetching chapter of Structured Streaming Kafka Integration Guide.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.maxBatchesToRetainInMemory",
+      "doc": "The maximum number of batches which will be retained in memory to avoid loading from files. The value adjusts a trade-off between memory usage vs cache miss: '2' covers both success and direct failure cases, '1' covers only success case, and '0' covers extreme case - disable cache to maximize memory size of executors.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.metadataCache.enabled",
+      "doc": "Whether the streaming HDFSMetadataLog caches the metadata of the latest two batches.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.metricsEnabled",
+      "doc": "Whether Dropwizard/Codahale metrics will be reported for active streaming queries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.minBatchesToRetain",
+      "doc": "The minimum number of batches that must be retained and made recoverable.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.streaming.multipleWatermarkPolicy",
+      "doc": "Policy to calculate the global watermark value when there are multiple watermark operators in a streaming query. The default value is 'min' which chooses the minimum watermark reported across multiple operators. Other alternative value is 'max' which chooses the maximum across multiple operators. Note: This configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "min"
+    },
+    {
+      "key": "spark.sql.streaming.noDataMicroBatches.enabled",
+      "doc": "Whether streaming micro-batch engine will execute batches without data for eager state management for stateful streaming queries.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.noDataProgressEventInterval",
+      "doc": "How long to wait before providing query idle event when there is no data",
+      "defaultValue": "10000ms"
+    },
+    {
+      "key": "spark.sql.streaming.numRecentProgressUpdates",
+      "doc": "The number of progress updates to retain for a streaming query",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.streaming.optimizeOneRowPlan.enabled",
+      "doc": "When true, enable OptimizeOneRowPlan rule for the case where the child is a streaming Dataset. This is a fallback flag to revert the 'incorrect' behavior, hence this configuration must not be used without understanding in depth. Use this only to quickly recover failure in existing query!",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.pollingDelay",
+      "doc": "How long to delay polling new data when no data is available",
+      "defaultValue": "10ms"
+    },
+    {
+      "key": "spark.sql.streaming.schemaInference",
+      "doc": "Whether file-based streaming sources will infer its own schema",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.sessionWindow.merge.sessions.in.local.partition",
+      "doc": "When true, streaming session window sorts and merge sessions in local partition prior to shuffle. This is to reduce the rows to shuffle, but only beneficial when there're lots of rows in a batch being assigned to same sessions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.sessionWindow.stateFormatVersion",
+      "doc": "State format version used by streaming session window in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.compression.codec",
+      "doc": "The codec used to compress delta and snapshot files generated by StateStore. By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify the codec. Default codec is lz4.",
+      "defaultValue": "lz4"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.formatValidation.enabled",
+      "doc": "When true, check if the data from state store is valid or not when running streaming queries. This can happen if the state store format has been changed. Note, the feature is only effective in the build-in HDFS state store provider now.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maintenanceInterval",
+      "doc": "The interval in milliseconds between triggering maintenance tasks in StateStore. The maintenance task executes background maintenance task in all the loaded store providers if they are still the active instances according to the coordinator. If not, inactive instances of store providers will be closed.",
+      "defaultValue": "60000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.minDeltasForSnapshot",
+      "doc": "Minimum number of state store delta files that needs to be generated before they consolidated into snapshots.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.providerClass",
+      "doc": "The class used to manage state data in stateful streaming queries. This class must be a subclass of StateStoreProvider, and must have a zero-arg constructor. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.rocksdb.formatVersion",
+      "doc": "Set the RocksDB format version. This will be stored in the checkpoint when starting a streaming query. The checkpoint will use this RocksDB format version in the entire lifetime of the query.",
+      "defaultValue": "5"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.skipNullsForStreamStreamJoins.enabled",
+      "doc": "When true, this config will skip null values in hash based stream-stream joins. The number of skipped null values will be shown as custom metric of stream join operator. If the streaming query was started with Spark 3.5 or above, please exercise caution before enabling this config since it may hide potential data loss/corruption issues.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.stateSchemaCheck",
+      "doc": "When true, Spark will validate the state schema against schema on existing state and fail query if it's incompatible.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.allowMultiple",
+      "doc": "When true, multiple stateful operators are allowed to be present in a streaming pipeline. The support for multiple stateful operators introduces a minor (semantically correct) change in respect to late record filtering - late records are detected and filtered in respect to the watermark from the previous microbatch instead of the current one. This is a behavior change for Spark streaming pipelines and we allow users to revert to the previous behavior of late record filtering (late records are detected and filtered by comparing with the current microbatch watermark) by setting the flag value to false. In this mode, only a single stateful operator will be allowed in a streaming pipeline.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.checkCorrectness.enabled",
+      "doc": "When true, the stateful operators for streaming query will be checked for possible correctness issue due to global watermark. The correctness issue comes from queries containing stateful operation which can emit rows older than the current watermark plus allowed late record delay, which are \"late rows\" in downstream stateful operations and these rows can be discarded. Please refer the programming guide doc for more details. Once the issue is detected, Spark will throw analysis exception. When this config is disabled, Spark will just print warning message for users. Prior to Spark 3.1.0, the behavior is disabling this config.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.useStrictDistribution",
+      "doc": "The purpose of this config is only compatibility; DO NOT MANUALLY CHANGE THIS!!! When true, the stateful operator for streaming query will use StatefulOpClusteredDistribution which guarantees stable state partitioning as long as the operator provides consistent grouping keys across the lifetime of query. When false, the stateful operator for streaming query will use ClusteredDistribution which is not sufficient to guarantee stable state partitioning despite the operator provides consistent grouping keys across the lifetime of query. This config will be set to true for new streaming queries to guarantee stable state partitioning, and set to false for existing streaming queries to not break queries which are restored from existing checkpoints. Please refer SPARK-38204 for details.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stopActiveRunOnRestart",
+      "doc": "Running multiple runs of the same streaming query concurrently is not supported. If we find a concurrent active run for a streaming query (in the same or different SparkSessions on the same cluster) and this flag is true, we will stop the old streaming query run to start the new one.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stopTimeout",
+      "doc": "How long to wait in milliseconds for the streaming execution thread to stop when calling the streaming query's stop() method. 0 or negative values wait indefinitely.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.streaming.streamingQueryListeners",
+      "doc": "List of class names implementing StreamingQueryListener that will be automatically added to newly created sessions. The classes should have either a no-arg constructor, or a constructor that expects a SparkConf argument.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.enabled",
+      "doc": "Whether to run the Structured Streaming Web UI for the Spark application when the Spark Web UI is enabled.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.enabledCustomMetricList",
+      "doc": "Configures a list of custom metrics on Structured Streaming UI, which are enabled. The list contains the name of the custom metrics separated by comma. In aggregationonly sum used. The list of supported custom metrics is state store provider specific and it can be found out for example from query progress log entry.",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.retainedProgressUpdates",
+      "doc": "The number of progress updates to retain for a streaming query for Structured Streaming UI.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.retainedQueries",
+      "doc": "The number of inactive queries to retain for Structured Streaming UI.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.unsupportedOperationCheck",
+      "doc": "When true, the logical plan for streaming query will be checked for unsupported operations.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.cache.maxEntries",
+      "doc": "The maximum entries of the cache used for interpreted subexpression elimination.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.enabled",
+      "doc": "When true, common subexpressions will be eliminated.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.skipForShortcutExpr",
+      "doc": "When true, shortcut eliminate subexpression with `AND`, `OR`. The subexpression may not need to eval even if it appears more than once. e.g., `if(or(a, and(b, b)))`, the expression `b` would be skipped if `a` is true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.subquery.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to execute the subquery.",
+      "defaultValue": "16",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.thriftServer.incrementalCollect",
+      "doc": "When true, enable incremental collection for execution in Thrift Server.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.thriftServer.interruptOnCancel",
+      "doc": "When true, all running tasks will be interrupted if one cancels a query. When false, all running tasks will remain until finished.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.thriftServer.queryTimeout",
+      "doc": "Set a query duration timeout in seconds in Thrift Server. If the timeout is set to a positive value, a running query will be cancelled automatically when the timeout is exceeded, otherwise the query continues to run till completion. If timeout values are set for each statement via `java.sql.Statement.setQueryTimeout` and they are smaller than this configuration value, they take precedence. If you set this timeout and prefer to cancel the queries right away without waiting task to finish, consider enabling spark.sql.thriftServer.interruptOnCancel together.",
+      "defaultValue": "0ms"
+    },
+    {
+      "key": "spark.sql.thriftserver.scheduler.pool",
+      "doc": "Set a Fair Scheduler pool for a JDBC client session."
+    },
+    {
+      "key": "spark.sql.thriftserver.ui.retainedSessions",
+      "doc": "The number of SQL client sessions kept in the JDBC/ODBC web UI history.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.thriftserver.ui.retainedStatements",
+      "doc": "The number of SQL statements kept in the JDBC/ODBC web UI history.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.timestampType",
+      "doc": "Configures the default timestamp type of Spark SQL, including SQL DDL, Cast clause, type literal and the schema inference of data sources. Setting the configuration as TIMESTAMP_NTZ will use TIMESTAMP WITHOUT TIME ZONE as the default type while putting it as TIMESTAMP_LTZ will use TIMESTAMP WITH LOCAL TIME ZONE. Before the 3.4.0 release, Spark only supports the TIMESTAMP WITH LOCAL TIME ZONE type.",
+      "defaultValue": "TIMESTAMP_LTZ"
+    },
+    {
+      "key": "spark.sql.truncateTable.ignorePermissionAcl.enabled",
+      "doc": "When set to true, TRUNCATE TABLE command will not try to set back original permission and ACLs when re-creating the table/partition paths.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.tvf.allowMultipleTableArguments.enabled",
+      "doc": "When true, allows multiple table arguments for table-valued functions, receiving the cartesian product of all the rows of these tables.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.udt.allowCreatingUDTFromString",
+      "doc": "When true, Spark loads and instantiates the UserDefinedType class named in a schema string (for example the schema stored in Parquet/ORC file metadata) while inferring or parsing a schema. Because the class name is taken from the data being read, a crafted file can make Spark load an arbitrary class from the classpath. Set this to false to block loading UDT classes by name, optionally allowing specific classes via 'spark.sql.udt.allowedDynamicUDTClasses'.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.udt.allowedDynamicUDTClasses",
+      "doc": "When 'spark.sql.udt.allowCreatingUDTFromString' is false, UserDefinedType classes listed here (by fully qualified class name) may still be loaded and instantiated from a schema string. Has no effect when UDT loading is enabled.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.ui.explainMode",
+      "doc": "Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.",
+      "defaultValue": "formatted"
+    },
+    {
+      "key": "spark.sql.ui.retainedExecutions",
+      "doc": "Number of executions to retain in the Spark UI.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.variable.substitute",
+      "doc": "This enables substitution using syntax like `${var}`, `${system:var}`, and `${env:var}`.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.view.maxNestedViewDepth",
+      "doc": "The maximum depth of a view reference in a nested view. A nested view may reference other nested views, the dependencies are organized in a directed acyclic graph (DAG). However the DAG depth may become too large and cause unexpected behavior. This configuration puts a limit on this: when the depth of a view exceeds this value during analysis, we terminate the resolution to avoid potential errors.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.warehouse.dir",
+      "doc": "The default location for managed databases and tables.",
+      "defaultValue": null,
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the window operator",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by window operator",
+      "defaultValue": "2147483647"
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/data/config/spark-4.0.json b/crates/sail-spark-connect/data/config/spark-4.0.json
new file mode 100644
index 000000000..ce3f0acca
--- /dev/null
+++ b/crates/sail-spark-connect/data/config/spark-4.0.json
@@ -0,0 +1,3291 @@
+{
+  "sparkVersion": "4.0.4",
+  "comment": "This is a generated file. Do not edit it directly.",
+  "entries": [
+    {
+      "key": "spark.connect.authenticate.token",
+      "doc": "A pre-shared token that will be used to authenticate clients. This secret must be passed as a bearer token by for clients to connect.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.copyFromLocalToFs.allowDestLocal",
+      "doc": "\n(Deprecated since Spark 4.0, please set\n'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead.\n",
+      "defaultValue": "false",
+      "isStatic": true,
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Use 'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead."
+      }
+    },
+    {
+      "key": "spark.connect.execute.manager.abandonedTombstonesSize",
+      "doc": "Maximum size of the cache of abandoned executions.",
+      "defaultValue": "10000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.detachedTimeout",
+      "doc": "Timeout after which executions without an attached RPC will be removed.",
+      "defaultValue": "5m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.maintenanceInterval",
+      "doc": "Interval at which execution manager will search for abandoned executions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.enabled",
+      "doc": "Enables reattachable execution on the server. If disabled and a client requests it, non-reattachable execution will follow and should run until query completion. This will work, unless there is a GRPC stream error, in which case the client will discover that execution is not reattachable when trying to reattach fails.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.observerRetryBufferSize",
+      "doc": "For reattachable execution, the total size of responses that were already sent to be kept in the buffer in case of connection error and client needing to retry. Set 0 to don't buffer anything (even last sent response).With any value greater than 0, the last sent response will always be buffered.",
+      "defaultValue": "10m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamDuration",
+      "doc": "For reattachable execution, after this amount of time the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "2m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamSize",
+      "doc": "For reattachable execution, after total responses size exceeds this value, the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "1g",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.command.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.CommandPlugin to support custom\nCommand types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.expression.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.ExpressionPlugin to support custom\nExpression types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.relation.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.RelationPlugin to support custom\nRelation types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.arrow.maxBatchSize",
+      "doc": "When using Apache Arrow, limit the maximum size of one arrow batch, in bytes unless otherwise specified, that can be sent from server side to client side. Currently, we conservatively use 70% of it because the size is not accurate but estimated.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.address",
+      "doc": "The address for Spark Connect server to bind.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.port",
+      "doc": "The port for Spark Connect server to bind.",
+      "defaultValue": "15002",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.interceptor.classes",
+      "doc": "Comma separated list of class names that must implement the io.grpc.ServerInterceptor interface.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.marshallerRecursionLimit",
+      "doc": "\nSets the recursion limit to grpc protobuf messages.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxInboundMessageSize",
+      "doc": "Sets the maximum inbound message in bytes size for the gRPC requests.Requests with a larger payload will fail.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxMetadataSize",
+      "doc": "Sets the maximum size of metadata fields. For instance, it restricts metadata fields in `ErrorInfo`.",
+      "defaultValue": "1024b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.port.maxRetries",
+      "doc": "The max port retry attempts for the gRPC server binding.By default, it's set to 0, and the server will fail fast in case of port conflicts.",
+      "defaultValue": "0",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.jvmStacktrace.maxSize",
+      "doc": "\nSets the maximum stack trace size to display when\n`spark.sql.pyspark.jvmStacktrace.enabled` is true.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.ml.backend.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.MLBackendPlugin to replace the\nspecified Spark ML operators with a backend-specific implementation.\n",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.connect.progress.reportInterval",
+      "doc": "The interval at which the progress of a query is reported to the client. If the value is set to a negative value the progress reports will be disabled.",
+      "defaultValue": "2s"
+    },
+    {
+      "key": "spark.connect.session.manager.closedSessionsTombstonesSize",
+      "doc": "Maximum size of the cache of sessions after which sessions that did not receive any requests will be removed.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.defaultSessionTimeout",
+      "doc": "Timeout after which sessions without any new incoming RPC will be removed. Setting it to -1 indicates that sessions should be kept forever.",
+      "defaultValue": "60m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.maintenanceInterval",
+      "doc": "Interval at which session manager will search for expired sessions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.planCache.enabled",
+      "doc": "When true, the cache of resolved logical plans is enabled if 'spark.connect.session.planCache.maxSize' is greater than zero. When false, the cache is disabled even if 'spark.connect.session.planCache.maxSize' is greater than zero. The caching is best-effort and not guaranteed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.planCache.maxSize",
+      "doc": "Sets the maximum number of cached resolved logical plans in Spark Connect Session. If set to a value less or equal than zero will disable the plan cache.",
+      "defaultValue": "16",
+      "isStatic": true
+    },
+    {
+      "key": "spark.python.sql.dataFrameDebugging.enabled",
+      "doc": "Enable the DataFrame debugging. This feature is enabled by default, but has a non-trivial performance overhead because of the stack trace collection.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.adaptive.advisoryPartitionSizeInBytes",
+      "doc": "The advisory size in bytes of the shuffle partition during adaptive optimization (when spark.sql.adaptive.enabled is true). It takes effect when Spark coalesces small shuffle partitions or splits skewed shuffle partition.",
+      "fallback": "spark.sql.adaptive.shuffle.targetPostShuffleInputSize"
+    },
+    {
+      "key": "spark.sql.adaptive.applyFinalStageShuffleOptimizations",
+      "doc": "Configures whether adaptive query execution (if enabled) should apply shuffle coalescing and local shuffle read optimization for the final query stage.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.autoBroadcastJoinThreshold",
+      "doc": "Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join. By setting this value to -1 broadcasting can be disabled. The default value is same with spark.sql.autoBroadcastJoinThreshold. Note that, this config is used only in adaptive framework."
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark will coalesce contiguous shuffle partitions according to the target size (specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes'), to avoid too many small tasks.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.initialPartitionNum",
+      "doc": "The initial number of shuffle partitions before coalescing. If not set, it equals to spark.sql.shuffle.partitions. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true."
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.minPartitionNum",
+      "doc": "(deprecated) The suggested (not guaranteed) minimum number of shuffle partitions after coalescing. If not set, the default value is the default parallelism of the Spark cluster. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true.",
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use 'spark.sql.adaptive.coalescePartitions.minPartitionSize' instead."
+      }
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.minPartitionSize",
+      "doc": "The minimum size of shuffle partitions after coalescing. This is useful when the adaptively calculated target size is too small during partition coalescing.",
+      "defaultValue": "1MB"
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.parallelismFirst",
+      "doc": "When true, Spark does not respect the target size specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes' (default 64MB) when coalescing contiguous shuffle partitions, but adaptively calculate the target size according to the default parallelism of the Spark cluster. The calculated size is usually smaller than the configured target size. This is to maximize the parallelism and avoid performance regressions when enabling adaptive query execution. It's recommended to set this config to false on a busy cluster to make resource utilization more efficient (not many small tasks).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.customCostEvaluatorClass",
+      "doc": "The custom cost evaluator class to be used for adaptive execution. If not being set, Spark will use its own SimpleCostEvaluator by default."
+    },
+    {
+      "key": "spark.sql.adaptive.enabled",
+      "doc": "When true, enable adaptive query execution, which re-optimizes the query plan in the middle of query execution, based on accurate runtime statistics.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.fetchShuffleBlocksInBatch",
+      "doc": "Whether to fetch the contiguous shuffle blocks in batch. Instead of fetching blocks one by one, fetching contiguous shuffle blocks for the same map task in batch can reduce IO and improve performance. Note, multiple contiguous blocks exist in single fetch request only happen when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true. This feature also depends on a relocatable serializer, the concatenation support codec in use, the new version shuffle fetch protocol and io encryption is disabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.forceApply",
+      "doc": "Adaptive query execution is skipped when the query does not have exchanges or sub-queries. By setting this config to true (together with 'spark.sql.adaptive.enabled' set to true), Spark will force apply adaptive query execution for all supported queries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.adaptive.forceOptimizeSkewedJoin",
+      "doc": "When true, force enable OptimizeSkewedJoin even if it introduces extra shuffle.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.adaptive.localShuffleReader.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark tries to use local shuffle reader to read the shuffle data when the shuffle partitioning is not needed, for example, after converting sort-merge join to broadcast-hash join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.logLevel",
+      "doc": "Configures the log level for adaptive execution logging of plan changes. The value can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.",
+      "defaultValue": "debug"
+    },
+    {
+      "key": "spark.sql.adaptive.maxShuffledHashJoinLocalMapThreshold",
+      "doc": "Configures the maximum size in bytes per partition that can be allowed to build local hash map. If this value is not smaller than spark.sql.adaptive.advisoryPartitionSizeInBytes and all the partition size are not larger than this config, join selection prefer to use shuffled hash join instead of sort merge join regardless of the value of spark.sql.join.preferSortMergeJoin.",
+      "defaultValue": "0b"
+    },
+    {
+      "key": "spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin",
+      "doc": "The relation with a non-empty partition ratio lower than this config will not be considered as the build side of a broadcast-hash join in adaptive execution regardless of its size.This configuration only has an effect when 'spark.sql.adaptive.enabled' is true.",
+      "defaultValue": "0.2"
+    },
+    {
+      "key": "spark.sql.adaptive.optimizeSkewsInRebalancePartitions.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark will optimize the skewed shuffle partitions in RebalancePartitions and split them to smaller ones according to the target size (specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes'), to avoid data skew.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.optimizer.excludedRules",
+      "doc": "Configures a list of rules to be disabled in the adaptive optimizer, in which the rules are specified by their rule names and separated by comma. The optimizer will log the rules that have indeed been excluded."
+    },
+    {
+      "key": "spark.sql.adaptive.rebalancePartitionsSmallPartitionFactor",
+      "doc": "A partition will be merged during splitting if its size is small than this factor multiply spark.sql.adaptive.advisoryPartitionSizeInBytes.",
+      "defaultValue": "0.2"
+    },
+    {
+      "key": "spark.sql.adaptive.shuffle.targetPostShuffleInputSize",
+      "doc": "(Deprecated since Spark 3.0)",
+      "defaultValue": "64MB",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.adaptive.advisoryPartitionSizeInBytes' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark dynamically handles skew in shuffled join (sort-merge and shuffled hash) by splitting (and replicating if needed) skewed partitions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.skewedPartitionFactor",
+      "doc": "A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than 'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'",
+      "defaultValue": "5.0"
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes",
+      "doc": "A partition is considered as skewed if its size in bytes is larger than this threshold and also larger than 'spark.sql.adaptive.skewJoin.skewedPartitionFactor' multiplying the median partition size. Ideally this config should be set larger than 'spark.sql.adaptive.advisoryPartitionSizeInBytes'.",
+      "defaultValue": "256MB"
+    },
+    {
+      "key": "spark.sql.addPartitionInBatch.size",
+      "doc": "The number of partitions to be handled in one turn when use `AlterTableAddPartitionCommand` or `RepairTableCommand` to add partitions into table. The smaller batch size is, the less memory is required for the real handler, e.g. Hive Metastore.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.allowNamedFunctionArguments",
+      "doc": "If true, Spark will turn on support for named parameters for all functions that has it implemented.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.alwaysInlineCommonExpr",
+      "doc": "When true, always inline common expressions instead of using the WITH expression. This may lead to duplicated expressions and the config should only be enabled if you hit bugs caused by the WITH expression.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.allowSubqueryExpressionsInLambdasOrHigherOrderFunctions",
+      "doc": "When set to false, the analyzer will throw an error if a subquery expression appears in a lambda function or higher-order function. When set to true, it restores the legacy behavior of allowing subquery eexpressions in lambda functions or higher-order functions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.canonicalization.multiCommutativeOpMemoryOptThreshold",
+      "doc": "The minimum number of operands in a commutative expression tree to invoke the MultiCommutativeOp memory optimization during canonicalization.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.analyzer.failAmbiguousSelfJoin",
+      "doc": "When true, fail the Dataset query if it contains ambiguous self-join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.maxIterations",
+      "doc": "The max number of iterations the analyzer runs.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.analyzer.scalarSubqueryAllowGroupByColumnEqualToConstant",
+      "doc": "When set to true, allow scalar subqueries with group-by on a column that also  has an equality filter with a constant (SPARK-48557).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.dualRunWithLegacy",
+      "doc": "When true, run both analyzers to check if single-pass Analyzer correctly produces the same analyzed plan as the fixed-point Analyzer for the existing set of features defined in the ResolverGuard",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.enabled",
+      "doc": "When true, use the single-pass Resolver instead of the fixed-point Analyzer. This is an alternative Analyzer framework, which resolves the parsed logical plan in a single post-order traversal. It uses ExpressionResolver to resolve expressions and NameScope to control the visibility of names. In contrast to the current fixed-point framework, subsequent in-tree traversals are disallowed. Most of the fixed-point Analyzer code is reused in the form of specific node transformation functions (AliasResolution.resolve, FunctionResolution.resolveFunction, etc). This feature is currently under development.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.relationBridging.enabled",
+      "doc": "When set to true, the single-pass Resolver will reuse the relation metadata that was previously resolved in fixed-point run. This makes sense only in ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER mode. In that case HybridAnalyzer enables the AnalyzerBridgeState and passes it to the single-pass Analyzer after the fixed-point run is complete. Single-pass Resolver uses this AnalyzerBridgeState to construct a special RelationMetadataProvider implementation - BridgedRelationMetadataProvider. This component simply reuses cached relation metadata and avoids any blocking calls (catalog RPCs or table metadata reads).",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.returnSinglePassResultInDualRun",
+      "doc": "When true, return the result of the single-pass resolver as the result of the dual run analysis (which is used if the ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER flag value is true). Otherwise, return the result of the fixed-point analyzer.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.validationEnabled",
+      "doc": "When true, validate the Resolver output with ResolutionValidator. The ResolutionValidator validates the resolved logical plan tree in one pass and asserts the internal contracts. It uses the ExpressionResolutionValidator internally to validate resolved expression trees in the same manner.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.sqlFunctionResolution.applyConfOverrides",
+      "doc": "When true, applies the conf overrides for certain feature flags during the resolution of user-defined sql table valued functions, consistent with view resolution.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.ansi.doubleQuotedIdentifiers",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, Spark SQL reads literals enclosed in double quoted (\") as identifiers. When false they are read as string literals.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.enabled",
+      "doc": "When true, Spark SQL uses an ANSI compliant dialect instead of being Hive compliant. For example, Spark will throw an exception at runtime instead of returning null results when the inputs to a SQL operator/function are invalid. For full details of this dialect, you can find them in the section \"ANSI Compliance\" of Spark's documentation. Some ANSI dialect features may be not from the ANSI SQL standard directly, but their behaviors align with ANSI SQL's style",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.ansi.enforceReservedKeywords",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, the Spark SQL parser enforces the ANSI reserved keywords and forbids SQL queries that use reserved keywords as alias names and/or identifiers for table, view, function, etc.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.relationPrecedence",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, JOIN takes precedence over comma when combining relation. For example, `t1, t2 JOIN t3` should result to `t1 X (t2 X t3)`. If the config is false, the result is `(t1 X t2) X t3`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.strictIndexOperator",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.4.0",
+        "comment": "This was an internal configuration. It is not needed anymore since Spark SQL always returns null when getting a map value with a non-existing key. See SPARK-40066 for more details."
+      }
+    },
+    {
+      "key": "spark.sql.artifact.copyFromLocalToFs.allowDestLocal",
+      "doc": "\nAllow `spark.copyFromLocalToFs` destination to be local file system\n path on spark driver node when\n`spark.sql.artifact.copyFromLocalToFs.allowDestLocal` is true.\nThis will allow user to overwrite arbitrary file on spark\ndriver node we should only enable it for testing purpose.\n"
+    },
+    {
+      "key": "spark.sql.artifact.isolation.alwaysApplyClassloader",
+      "doc": "When enabled, the classloader holding per-session artifacts will always be applied during SQL executions (useful for Spark Connect). When disabled, the classloader will be applied only when any artifact is added to the session.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.artifact.isolation.enabled",
+      "doc": "When enabled for a Spark Session, artifacts (such as JARs, files, archives) added to this session are isolated from other sessions within the same Spark instance. When disabled for a session, artifacts added to this session are visible to other sessions that have this config disabled. This config can only be set during the creation of a Spark Session and will have no effect when changed in the middle of session usage.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.assumeAnsiFalseIfNotPersisted.enabled",
+      "doc": "If enabled, assume ANSI mode is false if not persisted during view or UDF creation. Otherwise use the default value.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.autoBroadcastJoinThreshold",
+      "doc": "Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join. By setting this value to -1 broadcasting can be disabled.",
+      "defaultValue": "10MB"
+    },
+    {
+      "key": "spark.sql.avro.compression.codec",
+      "doc": "Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate, snappy, bzip2, xz and zstandard. Default codec is snappy.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.avro.datetimeRebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files. When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. When EXCEPTION, Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Avro files is unknown.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.avro.datetimeRebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files. When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. When EXCEPTION, Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.avro.deflate.level",
+      "doc": "Compression level for the deflate codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level in the current implementation.",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.avro.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to Avro datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.avro.xz.level",
+      "doc": "Compression level for the xz codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive The default value is 6.",
+      "defaultValue": "6"
+    },
+    {
+      "key": "spark.sql.avro.zstandard.bufferPool.enabled",
+      "doc": "If true, enable buffer pool of ZSTD JNI library when writing of AVRO files",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.avro.zstandard.level",
+      "doc": "Compression level for the zstandard codec used in writing of AVRO files. ",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.binaryOutputStyle",
+      "doc": "The output style used display binary data. Valid values are 'UTF-8', 'BASIC', 'BASE64', 'HEX', and 'HEX_DISCRETE'."
+    },
+    {
+      "key": "spark.sql.broadcastExchange.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to fetch and broadcast the table. If we encounter memory issue like frequently full GC or OOM when broadcast table we can decrease this number in order to reduce memory usage. Notice the number should be carefully chosen since decreasing parallelism might cause longer waiting for other broadcasting. Also, increasing parallelism may cause memory problem.",
+      "defaultValue": "128",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.broadcastTimeout",
+      "doc": "Timeout in seconds for the broadcast wait time in broadcast joins.",
+      "defaultValue": "300"
+    },
+    {
+      "key": "spark.sql.bucketing.coalesceBucketsInJoin.enabled",
+      "doc": "When true, if two bucketed tables with the different number of buckets are joined, the side with a bigger number of buckets will be coalesced to have the same number of buckets as the other side. Bigger number of buckets is divisible by the smaller number of buckets. Bucket coalescing is applied to sort-merge joins and shuffled hash join. Note: Coalescing bucketed table can avoid unnecessary shuffling in join, but it also reduces parallelism and could possibly cause OOM for shuffled hash join.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.bucketing.coalesceBucketsInJoin.maxBucketRatio",
+      "doc": "The ratio of the number of two buckets being coalesced should be less than or equal to this value for bucket coalescing to be applied. This configuration only has an effect when 'spark.sql.bucketing.coalesceBucketsInJoin.enabled' is set to true.",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.cache.serializer",
+      "doc": "The name of a class that implements org.apache.spark.sql.columnar.CachedBatchSerializer. It will be used to translate SQL data into a format that can more efficiently be cached. The underlying API is subject to change so use with caution. Multiple classes cannot be specified. The class must have a no-arg constructor.",
+      "defaultValue": "org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the cartesian product operator",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by cartesian product operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.caseSensitive",
+      "doc": "Whether the query analyzer should be case sensitive or not. Default to case insensitive. It is highly discouraged to turn on case sensitive mode.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.catalog.spark_catalog",
+      "doc": "A catalog implementation that will be used as the v2 interface to Spark's built-in v1 catalog: spark_catalog. This catalog shares its identifier namespace with the spark_catalog and must be consistent with it; for example, if a table can be loaded by the spark_catalog, this catalog must also return the table metadata. To delegate operations to the spark_catalog, implementations can extend 'CatalogExtension'. The value should be either 'builtin' which represents the spark's builit-in V2SessionCatalog, or a fully qualified class name of the catalog implementation.",
+      "defaultValue": "builtin"
+    },
+    {
+      "key": "spark.sql.catalog.spark_catalog.defaultDatabase",
+      "doc": "The default database for session catalog.",
+      "defaultValue": "default",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.catalogImplementation",
+      "doc": "",
+      "defaultValue": "in-memory",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.cbo.enabled",
+      "doc": "Enables CBO for estimation of plan statistics when set true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.card.weight",
+      "doc": "The weight of the ratio of cardinalities (number of rows) in the cost comparison function. The ratio of sizes in bytes has weight 1 - this value. The weighted geometric mean of these ratios is used to decide which of the candidate plans will be chosen by the CBO.",
+      "defaultValue": "0.7"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.dp.star.filter",
+      "doc": "Applies star-join filter heuristics to cost based join enumeration.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.dp.threshold",
+      "doc": "The maximum number of joined nodes allowed in the dynamic programming algorithm.",
+      "defaultValue": "12"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.enabled",
+      "doc": "Enables join reorder in CBO.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.planStats.enabled",
+      "doc": "When true, the logical plan will fetch row counts and column statistics from catalog.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.starJoinFTRatio",
+      "doc": "Specifies the upper limit of the ratio between the largest fact tables for a star join to be considered. ",
+      "defaultValue": "0.9"
+    },
+    {
+      "key": "spark.sql.cbo.starSchemaDetection",
+      "doc": "When true, it enables join reordering based on star schema detection. ",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.charAsVarchar",
+      "doc": "When true, Spark replaces CHAR type with VARCHAR type in CREATE/REPLACE/ALTER TABLE commands, so that newly created/updated tables will not have CHAR type columns/fields. Existing tables with CHAR type columns/fields are not affected by this config.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.chunkBase64String.enabled",
+      "doc": "Whether to truncate string generated by the `Base64` function. When true, base64 strings generated by the base64 function are chunked into lines of at most 76 characters. When false, the base64 strings are not chunked.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.cli.print.header",
+      "doc": "When set to true, spark-sql CLI prints the names of the columns in query output.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.fastHashMap.capacityBit",
+      "doc": "Capacity for the max number of rows to be held in memory by the fast hash aggregate product operator. The bit is not for actual value, but the actual numBuckets is determined by loadFactor (e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.twolevel.enabled",
+      "doc": "Enable two-level aggregate hash map. When enabled, records will first be inserted/looked-up at a 1st-level, small, fast map, and then fallback to a 2nd-level, larger, slower map when 1st level is full or keys cannot be found. When disabled, records go directly to the 2nd level.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.twolevel.partialOnly",
+      "doc": "Enable two-level aggregate hash map for partial aggregate only, because final aggregate might get more distinct keys compared to partial aggregate. Overhead of looking up 1st-level map might dominate when having a lot of distinct keys.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.vectorized.enable",
+      "doc": "Enable vectorized aggregate hash map. This is for testing/benchmarking only.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.sortAggregate.enabled",
+      "doc": "When true, enable code-gen for sort aggregate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.splitAggregateFunc.enabled",
+      "doc": "When true, the code generator would split aggregate code into individual methods instead of a single big method. This can be used to avoid oversized function that can miss the opportunity of JIT optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.broadcastCleanedSourceThreshold",
+      "doc": "A threshold (in string length) to determine if we should make the generated code a broadcast variable in whole stage codegen. To disable this, set the threshold to < 0; otherwise if the size is above the threshold, it'll use broadcast variable. Note that maximum string length allowed in Java is Integer.MAX_VALUE, so anything above it would be meaningless. The default value is set to -1 (disabled by default).",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.codegen.cache.maxEntries",
+      "doc": "When nonzero, enable caching of generated classes for operators and expressions. All jobs share the cache that can use up to the specified number for generated classes.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.codegen.comments",
+      "doc": "When true, put comment in the generated code. Since computing huge comments can be extremely expensive in certain cases, such as deeply-nested expressions which operate over inputs with wide schemas, default is false.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.codegen.factoryMode",
+      "doc": "This config determines the fallback behavior of several codegen generators during tests. `FALLBACK` means trying codegen first and then falling back to interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. `NO_CODEGEN` skips codegen and goes interpreted path always. Note that this configuration is only for the internal usage, and NOT supposed to be set by end users.",
+      "defaultValue": "FALLBACK"
+    },
+    {
+      "key": "spark.sql.codegen.fallback",
+      "doc": "When true, (whole stage) codegen could be temporary disabled for the part of query that fail to compile generated code",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.hugeMethodLimit",
+      "doc": "The maximum bytecode size of a single compiled Java function generated by whole-stage codegen. When the compiled function exceeds this threshold, the whole-stage codegen is deactivated for this subtree of the current query plan. The default value is 65535, which is the largest bytecode size possible for a valid Java method. When running on HotSpot, it may be preferable to set the value to 8000 to match HotSpot's implementation.",
+      "defaultValue": "65535"
+    },
+    {
+      "key": "spark.sql.codegen.join.buildSideOuterShuffledHashJoin.enabled",
+      "doc": "When true, enable code-gen for an OUTER shuffled hash join where outer side is the build side.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.existenceSortMergeJoin.enabled",
+      "doc": "When true, enable code-gen for Existence sort merge join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.fullOuterShuffledHashJoin.enabled",
+      "doc": "When true, enable code-gen for FULL OUTER shuffled hash join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.fullOuterSortMergeJoin.enabled",
+      "doc": "When true, enable code-gen for FULL OUTER sort merge join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.logging.maxLines",
+      "doc": "The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.codegen.maxFields",
+      "doc": "The maximum number of fields (including nested fields) that will be supported before deactivating whole-stage codegen.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.codegen.methodSplitThreshold",
+      "doc": "The threshold of source-code splitting in the codegen. When the number of characters in a single Java function (without comment) exceeds the threshold, the function will be automatically split to multiple smaller ones. We cannot know how many bytecode will be generated, so use the code length as metric. When running on HotSpot, a function's bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not be too small, otherwise there will be many function calls.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.codegen.splitConsumeFuncByOperator",
+      "doc": "When true, whole stage codegen would put the logic of consuming rows of each physical operator into individual methods, instead of a single big method. This can be used to avoid oversized function that can miss the opportunity of JIT optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.useIdInClassName",
+      "doc": "When true, embed the (whole-stage) codegen stage ID into the class name of the generated class as a suffix",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.wholeStage",
+      "doc": "When true, the whole stage (of multiple operators) will be compiled into single java method.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.columnNameOfCorruptRecord",
+      "doc": "The name of internal column for storing raw/un-parsed JSON and CSV records that fail to parse.",
+      "defaultValue": "_corrupt_record"
+    },
+    {
+      "key": "spark.sql.columnVector.offheap.enabled",
+      "doc": "When true, use OffHeapColumnVector in ColumnarBatch.",
+      "fallback": "spark.memory.offHeap.enabled"
+    },
+    {
+      "key": "spark.sql.connect.enrichError.enabled",
+      "doc": "\nWhen true, it enriches errors with full exception messages and optionally server-side\nstacktrace on the client side via an additional RPC.\n",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.connect.serverStacktrace.enabled",
+      "doc": "When true, it sets the server-side stacktrace in the user-facing Spark exception.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedSessions",
+      "doc": "The number of client sessions kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedStatements",
+      "doc": "The number of statements kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.constraintPropagation.enabled",
+      "doc": "When true, the query optimizer will infer and propagate data constraints in the query plan to optimize them. Constraint propagation can sometimes be computationally expensive for certain kinds of query plans (such as those with a large number of predicates and aliases) which might negatively impact overall runtime.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.crossJoin.enabled",
+      "doc": "When false, we will throw an error if a query contains a cartesian product without explicit CROSS JOIN syntax.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to CSV datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.parser.columnPruning.enabled",
+      "doc": "If it is set to true, column names of the requested schema are passed to CSV parser. Other column values can be ignored during parsing even if they are malformed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.parser.inputBufferSize",
+      "doc": "If it is set, it configures the buffer size of CSV input during parsing. It is the same as inputBufferSize option in CSV which has a higher priority. Note that this is a workaround for the parsing library's regression, and this configuration is internal and supposed to be removed in the near future."
+    },
+    {
+      "key": "spark.sql.dataFrameQueryContext.enabled",
+      "doc": "Enable the DataFrame query context. This feature is enabled by default, but has a non-trivial performance overhead because of the stack trace collection.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.dataSource.skipAssertOnPredicatePushdown",
+      "doc": "Enable skipping assert when expression in not translated to predicate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.dataframeCache.logLevel",
+      "doc": "Configures the log level of Dataframe cache operations, including adding and removing entries from Dataframe cache, hit and miss on cache application. The default log level is 'trace'. This log should only be used for debugging purposes and not in the production environment, since it generates a large amount of logs.",
+      "defaultValue": "trace"
+    },
+    {
+      "key": "spark.sql.datetime.java8API.enabled",
+      "doc": "If the configuration property is set to true, java.time.Instant and java.time.LocalDate classes of Java 8 API are used as external types for Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp and java.sql.Date are used for the same purpose.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.debug",
+      "doc": "Only used for internal debugging. Not all functions are supported when it is enabled.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.debug.maxToStringFields",
+      "doc": "Maximum number of fields of sequence-like entries can be converted to strings in debug output. Any elements beyond the limit will be dropped and replaced by a  \"... N more fields\" placeholder.",
+      "defaultValue": "25"
+    },
+    {
+      "key": "spark.sql.decimalOperations.allowPrecisionLoss",
+      "doc": "When true (default), establishing the result type of an arithmetic operation happens according to Hive behavior and SQL ANSI 2011 specification, i.e. rounding the decimal part of the result if an exact representation is not possible. Otherwise, NULL is returned in those cases, as previously.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultCacheStorageLevel",
+      "doc": "The default storage level of `dataset.cache()`, `catalog.cacheTable()` and sql query `CACHE TABLE t`.",
+      "defaultValue": "MEMORY_AND_DISK"
+    },
+    {
+      "key": "spark.sql.defaultCatalog",
+      "doc": "Name of the default catalog. This will be the current catalog if users have not explicitly set the current catalog yet.",
+      "defaultValue": "spark_catalog"
+    },
+    {
+      "key": "spark.sql.defaultColumn.allowedProviders",
+      "doc": "List of table providers wherein SQL commands are permitted to assign DEFAULT column values. Comma-separated list, whitespace ignored, case-insensitive. If an asterisk appears after any table provider in this list, any command may assign DEFAULT column except `ALTER TABLE ... ADD COLUMN`. Otherwise, if no asterisk appears, all commands are permitted. This is useful because in order for such `ALTER TABLE ... ADD COLUMN` commands to work, the target data source must include support for substituting in the provided values when the corresponding fields are not present in storage.",
+      "defaultValue": "csv,json,orc,parquet"
+    },
+    {
+      "key": "spark.sql.defaultColumn.enabled",
+      "doc": "When true, allow CREATE TABLE, REPLACE TABLE, and ALTER COLUMN statements to set or update default values for specific columns. Following INSERT, MERGE, and UPDATE statements may then omit these values and their values will be injected automatically instead.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultColumn.useNullsForMissingDefaultValues",
+      "doc": "When true, and DEFAULT columns are enabled, allow INSERT INTO commands with user-specified lists of fewer columns than the target table to behave as if they had specified DEFAULT for all remaining columns instead, in order.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultSizeInBytes",
+      "doc": "The default table size used in query planning. By default, it is set to Long.MaxValue which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. That is to say by default the optimizer will not choose to broadcast a table unless it knows for sure its size is small enough.",
+      "defaultValue": "9223372036854775807b"
+    },
+    {
+      "key": "spark.sql.defaultUrlStreamHandlerFactory.enabled",
+      "doc": "When true, register Hadoop's FsUrlStreamHandlerFactory to support ADD JAR against HDFS locations. It should be disabled when a different stream protocol handler should be registered to support a particular protocol type, or if Hadoop's FsUrlStreamHandlerFactory conflicts with other protocol types such as `http` or `https`. See also SPARK-25694 and HADOOP-14598.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.error.messageFormat",
+      "doc": "When PRETTY, the error message consists of textual representation of error class, message and query context. The MINIMAL and STANDARD formats are pretty JSON formats where STANDARD includes an additional JSON field `message`. This configuration property influences on error messages of Thrift Server and SQL CLI while running queries.",
+      "defaultValue": "PRETTY"
+    },
+    {
+      "key": "spark.sql.event.truncate.length",
+      "doc": "Threshold of SQL length beyond which it will be truncated before adding to event. Defaults to no truncation. If set to 0, callsite will be logged instead.",
+      "defaultValue": "2147483647",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.exchange.reuse",
+      "doc": "When true, the planner will try to find out duplicated exchanges and re-use them.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.arrow.enabled",
+      "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.execution.arrow.pyspark.enabled' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.execution.arrow.fallback.enabled",
+      "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.fallback.enabled'.)",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.execution.arrow.pyspark.fallback.enabled' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.execution.arrow.localRelationThreshold",
+      "doc": "When converting Arrow batches to Spark DataFrame, local collections are used in the driver side if the byte size of Arrow batches is smaller than this threshold. Otherwise, the Arrow batches are sent and deserialized to Spark internal rows in the executors.",
+      "defaultValue": "48MB"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxBytesPerBatch",
+      "doc": "When using Apache Arrow, limit the maximum bytes in each batch that can be written to a single ArrowRecordBatch in memory. This configuration is not effective for the grouping API such as DataFrame(.cogroup).groupby.applyInPandas because each group becomes each ArrowRecordBatch. Unlike 'spark.sql.execution.arrow.maxRecordsPerBatch', this configuration does not work for createDataFrame/toPandas with Arrow/pandas instances. See also spark.sql.execution.arrow.maxRecordsPerBatch. If both are set, each batch is created when any condition of both is met.",
+      "defaultValue": "256MB"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxRecordsPerBatch",
+      "doc": "When using Apache Arrow, limit the maximum number of records that can be written to a single ArrowRecordBatch in memory. This configuration is not effective for the grouping API such as DataFrame(.cogroup).groupby.applyInPandas because each group becomes each ArrowRecordBatch. If set to zero or negative there is no limit. See also spark.sql.execution.arrow.maxBytesPerBatch. If both are set, each batch is created when any condition of both is met.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.enabled",
+      "doc": "When true, make use of Apache Arrow for columnar data transfers in PySpark. This optimization applies to: 1. pyspark.sql.DataFrame.toPandas. 2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame or a NumPy ndarray. The following data type is unsupported: ArrayType of TimestampType.",
+      "fallback": "spark.sql.execution.arrow.enabled"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.fallback.enabled",
+      "doc": "When true, optimizations enabled by 'spark.sql.execution.arrow.pyspark.enabled' will fallback automatically to non-optimized implementations if an error occurs.",
+      "fallback": "spark.sql.execution.arrow.fallback.enabled"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.selfDestruct.enabled",
+      "doc": "(Experimental) When true, make use of Apache Arrow's self-destruct and split-blocks options for columnar data transfers in PySpark, when converting from Arrow to Pandas. This reduces memory usage at the cost of some CPU time. This optimization applies to: pyspark.sql.DataFrame.toPandas when 'spark.sql.execution.arrow.pyspark.enabled' is set.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.sparkr.enabled",
+      "doc": "When true, make use of Apache Arrow for columnar data transfers in SparkR. This optimization applies to: 1. createDataFrame when its input is an R DataFrame 2. collect 3. dapply 4. gapply The following data types are unsupported: FloatType, BinaryType, ArrayType, StructType and MapType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.transformWithStateInPandas.maxRecordsPerBatch",
+      "doc": "When using TransformWithStateInPandas, limit the maximum number of state records that can be written to a single ArrowRecordBatch in memory.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.execution.arrow.useLargeVarTypes",
+      "doc": "When using Apache Arrow, use large variable width vectors for string and binary types. Regular string and binary types have a 2GiB limit for a column in a single record batch. Large variable types remove this limitation at the cost of higher memory usage per value.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.broadcastHashJoin.outputPartitioningExpandLimit",
+      "doc": "The maximum number of partitionings that a HashPartitioning can be expanded to. This configuration is applicable only for BroadcastHashJoin inner joins and can be set to '0' to disable this feature.",
+      "defaultValue": "8"
+    },
+    {
+      "key": "spark.sql.execution.fastFailOnFileFormatOutput",
+      "doc": "Whether to fast fail task execution when writing output to FileFormat datasource. If this is enabled, in `FileFormatWriter` we will catch `FileAlreadyExistsException` and fast fail output task without further task retry. Only enabling this if you know the `FileAlreadyExistsException` of the output task is unrecoverable, i.e., further task attempts won't be able to success. If the `FileAlreadyExistsException` might be recoverable, you should keep this as disabled and let Spark to retry output tasks. This is disabled by default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.interruptOnCancel",
+      "doc": "When true, all running tasks will be interrupted if one cancels a query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pandas.convertToArrowArraySafely",
+      "doc": "When true, Arrow will perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. When false, disabling Arrow's type check and do type conversions anyway. This config only works for Arrow 0.11.0+.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pandas.inferPandasDictAsMap",
+      "doc": "When true, spark.createDataFrame will infer dict from Pandas DataFrame as a MapType. When false, spark.createDataFrame infers dict from Pandas DataFrame as a StructType which is default inferring from PyArrow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pandas.respectSessionTimeZone",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "The non-default behavior is considered as a bug, see SPARK-22395. The config was deprecated since Spark 2.3."
+      }
+    },
+    {
+      "key": "spark.sql.execution.pandas.structHandlingMode",
+      "doc": "The conversion mode of struct type when creating pandas DataFrame. When \"legacy\", 1. when Arrow optimization is disabled, convert to Row object, 2. when Arrow optimization is enabled, convert to dict or raise an Exception if there are duplicated nested field names. When \"row\", convert to Row object regardless of Arrow optimization. When \"dict\", convert to dict and use suffixed key names, e.g., a_0, a_1, if there are duplicated nested field names, regardless of Arrow optimization.",
+      "defaultValue": "legacy"
+    },
+    {
+      "key": "spark.sql.execution.pandas.udf.buffer.size",
+      "doc": "Same as `spark.buffer.size` but only applies to Pandas UDF executions. If it is not set, the fallback is `spark.buffer.size`. Note that Pandas execution requires more than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and pipelined; however, it might degrade performance. See SPARK-27870.",
+      "fallback": "spark.buffer.size"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.python",
+      "doc": "Python binary executable to use for PySpark in executors when running Python UDF, pandas UDF and pandas function APIs. If not set, it falls back to 'spark.pyspark.python' by default."
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.faulthandler.enabled",
+      "doc": "Same as spark.python.worker.faulthandler.enabled for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.worker.faulthandler.enabled"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.hideTraceback.enabled",
+      "doc": "When true, only show the message of the exception from Python UDFs, hiding the stack trace. If this is enabled, simplifiedTraceback has no effect.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.idleTimeoutSeconds",
+      "doc": "Same as spark.python.worker.idleTimeoutSeconds for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.worker.idleTimeoutSeconds"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled",
+      "doc": "When true, the traceback from Python UDFs is simplified. It hides the Python worker, (de)serialization, etc from PySpark in tracebacks, and only shows the exception messages from UDFs. Note that this works only with CPython 3.7+.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.python.udf.buffer.size",
+      "doc": "Same as `spark.buffer.size` but only applies to Python UDF executions. If it is not set, the fallback is `spark.buffer.size`.",
+      "fallback": "spark.buffer.size"
+    },
+    {
+      "key": "spark.sql.execution.python.udf.maxRecordsPerBatch",
+      "doc": "When using Python UDFs, limit the maximum number of records that can be batched for serialization/deserialization.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.arrow.concurrency.level",
+      "doc": "The level of concurrency to execute Arrow-optimized Python UDF. This can be useful if Python UDFs use I/O intensively."
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.arrow.enabled",
+      "doc": "Enable Arrow optimization in regular Python UDFs. This optimization can only be enabled when the given function takes at least one argument.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDTF.arrow.enabled",
+      "doc": "Enable Arrow optimization for Python UDTFs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.rangeExchange.sampleSizePerPartition",
+      "doc": "Number of points to sample per partition in order to determine the range boundaries for range partitioning, typically used in global sorting (without limit).",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.execution.removeRedundantProjects",
+      "doc": "Whether to remove redundant project exec node based on children's output and ordering requirement.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.removeRedundantSorts",
+      "doc": "Whether to remove redundant physical sort node",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.replaceHashWithSortAgg",
+      "doc": "Whether to replace hash aggregate node with sort aggregate based on children's ordering",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.reuseSubquery",
+      "doc": "When true, the planner will try to find out duplicated subqueries and re-use them.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.sortBeforeRepartition",
+      "doc": "When perform a repartition following a shuffle, the output row ordering would be nondeterministic. If some downstream stages fail and some tasks of the repartition stage retry, these tasks may generate different data, and that can lead to correctness issues. Turn on this config to insert a local sort before actually doing repartition to generate consistent repartition results. The performance of repartition() may go down since we insert extra local sort before it.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.topKSortFallbackThreshold",
+      "doc": "In SQL queries with a SORT followed by a LIMIT like 'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort in memory, otherwise do a global sort which spills to disk if necessary.",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.execution.useObjectHashAggregateExec",
+      "doc": "Decides if we use ObjectHashAggregateExec",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.usePartitionEvaluator",
+      "doc": "When true, use PartitionEvaluator to execute SQL operators.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.expressionTreeChangeLog.level",
+      "doc": "Configures the log level for logging the change from the unresolved expression tree to the resolved expression tree in the single-pass bottom-up Resolver. The value can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'trace'.",
+      "defaultValue": "trace"
+    },
+    {
+      "key": "spark.sql.extendedExplainProviders",
+      "doc": "A comma-separated list of classes that implement the org.apache.spark.sql.ExtendedExplainGenerator trait. If provided, Spark will print extended plan information from the providers in explain plan and in the UI"
+    },
+    {
+      "key": "spark.sql.extensions",
+      "doc": "A comma-separated list of classes that implement Function1[SparkSessionExtensions, Unit] used to configure Spark Session extensions. The classes must have a no-args constructor. If multiple extensions are specified, they are applied in the specified order. For the case of rules and planner strategies, they are applied in the specified order. For the case of parsers, the last parser is used and each parser can delegate to its predecessor. For the case of function name conflicts, the last registered function name is used.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.extensions.test.loadFromCp",
+      "doc": "Flag that determines if we should load extensions from the classpath using the SparkSessionExtensionsProvider mechanism. This is a test only flag.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.files.ignoreCorruptFiles",
+      "doc": "Whether to ignore corrupt files. If true, the Spark jobs will continue to run when encountering corrupted files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.ignoreInvalidPartitionPaths",
+      "doc": "Whether to ignore invalid partition paths that do not match =. When the option is enabled, table with two partition directories 'table/invalid' and 'table/col=1' will only load the latter directory and ignore the invalid partition",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.ignoreMissingFiles",
+      "doc": "Whether to ignore missing files. If true, the Spark jobs will continue to run when encountering missing files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.maxPartitionBytes",
+      "doc": "The maximum number of bytes to pack into a single partition when reading files. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "128MB"
+    },
+    {
+      "key": "spark.sql.files.maxPartitionNum",
+      "doc": "The suggested (not guaranteed) maximum number of split file partitions. If it is set, Spark will rescale each partition to make the number of partitions is close to this value if the initial number of partitions exceeds this value. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC."
+    },
+    {
+      "key": "spark.sql.files.maxRecordsPerFile",
+      "doc": "Maximum number of records to write out to a single file. If this value is zero or negative, there is no limit.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.files.minPartitionNum",
+      "doc": "The suggested (not guaranteed) minimum number of split file partitions. If not set, the default value is `spark.sql.leafNodeDefaultParallelism`. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC."
+    },
+    {
+      "key": "spark.sql.files.openCostInBytes",
+      "doc": "The estimated cost to open a file, measured by the number of bytes could be scanned in the same time. This is used when putting multiple files into a partition. It's better to over estimated, then the partitions with small files will be faster than partitions with bigger files (which is scheduled first). This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "4MB"
+    },
+    {
+      "key": "spark.sql.files.supportSecondOffsetFormat",
+      "doc": "When set to true, datetime formatter used for csv, json and xml will support zone offsets that have seconds in it. e.g. LA timezone offset prior to 1883 was -07:52:58. When this flag is not set we lose seconds information.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.filesourceTableRelationCacheSize",
+      "doc": "The maximum size of the cache that maps qualified table names to table relation plans.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.fromJsonForceNullableSchema",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent errors like SPARK-23173 for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.function.concatBinaryAsString",
+      "doc": "When this option is set to false and all inputs are binary, `functions.concat` returns an output as binary. Otherwise, it returns as a string.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.function.eltOutputAsString",
+      "doc": "When this option is set to false and all inputs are binary, `elt` returns an output as binary. Otherwise, it returns as a string.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.globalTempDatabase",
+      "doc": "",
+      "defaultValue": "global_temp",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.groupByAliases",
+      "doc": "When true, aliases in a select list can be used in group by clauses. When false, an analysis exception is thrown in the case.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.groupByOrdinal",
+      "doc": "When true, the ordinal numbers in group by clauses are treated as the position in the select list. When false, the ordinal numbers are ignored.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.advancedPartitionPredicatePushdown.enabled",
+      "doc": "When true, advanced partition predicate pushdown into Hive metastore is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.caseSensitiveInferenceMode",
+      "doc": "Sets the action to take when a case-sensitive schema cannot be read from a Hive Serde table's properties when reading the table with Spark native data sources. Valid options include INFER_AND_SAVE (infer the case-sensitive schema from the underlying data files and write it back to the table properties), INFER_ONLY (infer the schema but don't attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback to using the case-insensitive metastore schema instead of inferring).",
+      "defaultValue": "NEVER_INFER"
+    },
+    {
+      "key": "spark.sql.hive.convertCTAS",
+      "doc": "When true, a table created by a Hive CTAS statement (no USING clause) without specifying any storage property will be converted to a data source table, using the data source set by spark.sql.sources.default.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.1",
+        "comment": "Set 'spark.sql.legacy.createHiveTableByDefault' to false instead."
+      }
+    },
+    {
+      "key": "spark.sql.hive.dropPartitionByName.enabled",
+      "doc": "When true, Spark will get partition name rather than partition object to drop partition, which can improve the performance of drop partition.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.filesourcePartitionFileCacheSize",
+      "doc": "When nonzero, enable caching of partition file metadata in memory. All tables share a cache that can use up to specified num bytes for file metadata. This conf only has an effect when hive filesource partition management is enabled.",
+      "defaultValue": "262144000"
+    },
+    {
+      "key": "spark.sql.hive.gatherFastStats",
+      "doc": "When true, fast stats (number of files and total size of all files) will be gathered in parallel while repairing table partitions to avoid the sequential listing in Hive metastore.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.manageFilesourcePartitions",
+      "doc": "When true, enable metastore partition management for file source tables as well. This includes both datasource and converted Hive tables. When partition management is enabled, datasource tables store partition in the Hive metastore, and use the metastore to prune partitions during query planning when spark.sql.hive.metastorePartitionPruning is set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruning",
+      "doc": "When true, some predicates will be pushed down into the Hive metastore so that unmatching partitions can be eliminated earlier.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningFallbackOnException",
+      "doc": "Whether to fallback to get all partitions from Hive metastore and perform partition pruning on Spark client side, when encountering MetaException from the metastore. Note that Spark query performance may degrade if this is enabled and there are many partitions to be listed. If this is disabled, Spark will fail the query instead.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningFastFallback",
+      "doc": "When this config is enabled, if the predicates are not supported by Hive or Spark does fallback due to encountering MetaException from the metastore, Spark will instead prune partitions by getting the partition names first and then evaluating the filter expressions on the client side. Note that the predicates with TimeZoneAwareExpression is not supported.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningInSetThreshold",
+      "doc": "The threshold of set size for InSet predicate when pruning partitions through Hive Metastore. When the set size exceeds the threshold, we rewrite the InSet predicate to be greater than or equal to the minimum value in set and less than or equal to the maximum value in set. Larger values may cause Hive Metastore stack overflow. But for InSet inside Not with values exceeding the threshold, we won't push it to Hive Metastore.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.hive.tablePropertyLengthThreshold",
+      "doc": "The maximum length allowed in a single cell when storing Spark-specific information in Hive's metastore as table properties. Currently it covers 2 things: the schema's JSON string, the histogram of column statistics."
+    },
+    {
+      "key": "spark.sql.hive.thriftServer.singleSession",
+      "doc": "When set to true, Hive Thrift server is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.hive.verifyPartitionPath",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "This config was replaced by 'spark.sql.files.ignoreMissingFiles'."
+      }
+    },
+    {
+      "key": "spark.sql.icu.caseMappings.enabled",
+      "doc": "When enabled we use the ICU library (instead of the JVM) to implement case mappings for strings under UTF8_BINARY collation.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.batchSize",
+      "doc": "Controls the size of batches for columnar caching.  Larger batch sizes can improve memory utilization and compression, but risk OOMs when caching data.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.compressed",
+      "doc": "When set to true Spark SQL will automatically select a compression codec for each column based on statistics of the data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.enableVectorizedReader",
+      "doc": "Enables vectorized reader for columnar caching.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.hugeVectorReserveRatio",
+      "doc": "When spark.sql.inMemoryColumnarStorage.hugeVectorThreshold <= 0 or the required memory is smaller than spark.sql.inMemoryColumnarStorage.hugeVectorThreshold, spark reserves required memory * 2 memory; otherwise, spark reserves required memory * this ratio memory, and will release this column vector memory before reading the next batch rows.",
+      "defaultValue": "1.2"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.hugeVectorThreshold",
+      "doc": "When the required memory is larger than this, spark reserves required memory * spark.sql.inMemoryColumnarStorage.hugeVectorReserveRatio memory next time and release this column vector memory before reading the next batch rows. -1 means disabling the optimization.",
+      "defaultValue": "-1b"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.partitionPruning",
+      "doc": "When true, enable partition pruning for in-memory columnar tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryTableScanStatistics.enable",
+      "doc": "When true, enable in-memory table scan accumulators.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.join.preferSortMergeJoin",
+      "doc": "When true, prefer sort merge join over shuffled hash join. Sort merge join consumes less memory than shuffled hash join and it works efficiently when both join tables are large. On the other hand, shuffled hash join can improve performance (e.g., of full outer joins) when one of join tables is much smaller.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.enableExactStringParsing",
+      "doc": "When set to true, string columns extracted from JSON objects will be extracted exactly as they appear in the input string, with no changes",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.enablePartialResults",
+      "doc": "When set to true, enables partial results for structs, maps, and arrays in JSON when one or more fields do not match the schema",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to JSON datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.useUnsafeRow",
+      "doc": "When set to true, use UnsafeRow to represent struct result in the JSON parser. It can be overwritten by the JSON option `useUnsafeRow`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.jsonGenerator.ignoreNullFields",
+      "doc": "Whether to ignore null fields when generating JSON objects in JSON data source and JSON functions such as to_json. If false, it generates null for null fields in JSON objects.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.jsonGenerator.writeNullIfWithDefaultValue",
+      "doc": "When true, when writing NULL values to columns of JSON tables with explicit DEFAULT values using INSERT, UPDATE, or MERGE commands, never skip writing the NULL values to storage, overriding spark.sql.jsonGenerator.ignoreNullFields or the ignoreNullFields option. This can be useful to enforce that inserted NULL values are present in storage to differentiate from missing data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lateralColumnAlias.enableImplicitResolution",
+      "doc": "Enable resolving implicit lateral column alias defined in the same SELECT list. For example, with this conf turned on, for query `SELECT 1 AS a, a + 1` the `a` in `a + 1` can be resolved as the previously defined `1 AS a`. But note that table column has higher resolution priority than the lateral column alias.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lazySetOperatorOutput.enabled",
+      "doc": "When set to true, Except/Intersect/Union operator's output will be a lazy val. It is a performance optimization for querires with a large number of stacked set operators. This is because of rules like WidenSetOperationTypes that traverse the logical plan tree and call output on each Except/Intersect/Union node. Such traversal has quadratic complexity: O(number_of_nodes * (1 + 2 + 3  + ... + number_of_nodes)).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.leafNodeDefaultParallelism",
+      "doc": "The default parallelism of Spark SQL leaf nodes that produce data, such as the file scan node, the local data scan node, the range node, etc. The default value of this config is 'SparkContext#defaultParallelism'."
+    },
+    {
+      "key": "spark.sql.legacy.addSingleFileInAddFile",
+      "doc": "When true, only a single file can be added using ADD FILE. If false, then users can add directory by passing directory path to ADD FILE.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowAutoGeneratedAliasForView",
+      "doc": "When true, it's allowed to use a input query without explicit alias when creating a permanent view.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent loss of user data for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.allowEmptySchemaWrite",
+      "doc": "When this option is set to true, validation of empty or empty nested schemas that occurs when writing into a FileFormat based data source does not happen.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowHashOnMapType",
+      "doc": "When set to true, hash expressions can be applied on elements of MapType. Otherwise, an analysis exception will be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNegativeScaleOfDecimal",
+      "doc": "When set to true, negative scale of Decimal type is allowed. For example, the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is Decimal(11, 0) in non legacy mode.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNonEmptyLocationInCTAS",
+      "doc": "When false, CTAS with LOCATION throws an analysis exception if the location is not empty.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNullComparisonResultInArraySort",
+      "doc": "When set to false, `array_sort` function throws an error if the comparator function returns null. If set to true, it restores the legacy behavior that handles null as zero (equal).",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowParameterlessCount",
+      "doc": "When true, the SQL function 'count' is allowed to take no parameters.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowStarWithSingleTableIdentifierInCount",
+      "doc": "When true, the SQL function 'count' is allowed to take single 'tblName.*' as parameter",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowTempViewCreationWithMultipleNameparts",
+      "doc": "When true, temp view creation Dataset APIs will allow the view creation even if the view name is multiple name parts. The extra name parts will be dropped during the view creation",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowUntypedScalaUDF",
+      "doc": "When set to true, user is allowed to use org.apache.spark.sql.functions. udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowZeroIndexInFormatString",
+      "doc": "When false, the `strfmt` in `format_string(strfmt, obj, ...)` and `printf(strfmt, obj, ...)` will no longer support to use \"0$\" to specify the first argument, the first argument should always reference by \"1$\" when use argument index to indicating the position of the argument in the argument list. This config will be removed in the future releases.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Increase indexes by 1 in `strfmt` of the `format_string` function. Refer to the first argument by \"1$\"."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.avro.allowIncompatibleSchema",
+      "doc": "When set to false, if types in Avro are encoded in the same format, but the type in the Avro schema explicitly says that the data types are different, reject reading the data type in the format to avoid returning incorrect results. When set to true, it restores the legacy behavior of allow reading the data in the format, which may return incorrect results.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.avro.datetimeRebaseModeInRead",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.avro.datetimeRebaseModeInRead' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.avro.datetimeRebaseModeInWrite",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.avro.datetimeRebaseModeInWrite' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.bangEqualsNot",
+      "doc": "When set to true, '!' is a lexical equivalent for 'NOT'. That is '!' can be used outside of the documented prefix usage in a logical expression. Examples are: `expr ! IN (1, 2)` and `expr ! BETWEEN 1 AND 2`, but also `IF ! EXISTS`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.bucketedTableScan.outputOrdering",
+      "doc": "When true, the bucketed table scan will list files during planning to figure out the output ordering, which is expensive and may make the planning quite slow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.castComplexTypesToString.enabled",
+      "doc": "When true, maps and structs are wrapped by [] in casting to strings, and NULL elements of structs/maps/arrays will be omitted while converting to strings. Otherwise, if this is false, which is the default, maps and structs are wrapped by {}, and NULL elements will be converted to \"null\".",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.charVarcharAsString",
+      "doc": "When true, Spark treats CHAR/VARCHAR type the same as STRING type, which is the behavior of Spark 3.0 and earlier. This means no length check for CHAR/VARCHAR type and no padding for CHAR type when writing data to the table.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.codingErrorAction",
+      "doc": "When set to true, encode/decode functions replace unmappable characters with mojibake instead of reporting coding errors.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.compareDateTimestampInTimestamp",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent errors like SPARK-23549 for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.createEmptyCollectionUsingStringType",
+      "doc": "When set to true, Spark returns an empty collection with `StringType` as element type if the `array`/`map` function is called without any parameters. Otherwise, Spark returns an empty collection with `NullType` as element type.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.createHiveTableByDefault",
+      "doc": "When set to true, CREATE TABLE syntax without USING or STORED AS will use Hive instead of the value of spark.sql.sources.default as the table provider.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.csv.enableDateTimeParsingFallback",
+      "doc": "When true, enable legacy date/time parsing fallback in CSV"
+    },
+    {
+      "key": "spark.sql.legacy.ctePrecedencePolicy",
+      "doc": "When LEGACY, outer CTE definitions takes precedence over inner definitions. If set to EXCEPTION, AnalysisException is thrown while name conflict is detected in nested CTE. The default is CORRECTED, inner CTE definitions take precedence. This config will be removed in future versions and CORRECTED will be the only behavior.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.legacy.dataFrameWriterV2IgnorePathOption",
+      "doc": "When set to true, DataFrameWriterV2 ignores the 'path' option and always write data to the default table location.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue",
+      "doc": "When set to true, the key attribute resulted from running `Dataset.groupByKey` for non-struct key type, will be named as `value`, following the behavior of Spark version 2.4 and earlier.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.db2.booleanMapping.enabled",
+      "doc": "When true, BooleanType maps to CHAR(1) in DB2; otherwise, BOOLEAN",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.db2.numericMapping.enabled",
+      "doc": "When true, SMALLINT maps to IntegerType in DB2; otherwise, ShortType",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.decimal.retainFractionDigitsOnTruncate",
+      "doc": "When set to true, we will try to retain the fraction digits first rather than integral digits as prior Spark 4.0, when getting a least common type between decimal types, and the result decimal precision exceeds the max precision.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.disableMapKeyNormalization",
+      "doc": "Disables key normalization when creating a map with `ArrayBasedMapBuilder`. When set to `true` it will prevent key normalization when building a map, which will allow for values such as `-0.0` and `0.0` to be present as distinct keys.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.doLooseUpcast",
+      "doc": "When true, the upcast will be loose and allows string to atomic types.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.duplicateBetweenInput",
+      "doc": "When true, we use legacy between implementation. This is a flag that fixes a problem introduced by a between optimization, see ticket SPARK-49063.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.earlyEvalCurrentTime",
+      "doc": "When set to true, evaluation and constant folding will happen for now() and current_timestamp() expressions before finish analysis phase. This flag will allow a bit more liberal syntax but it will sacrifice correctness - Results of now() and current_timestamp() can be different for different operations in a single query.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.emptyCurrentDBInCli",
+      "doc": "When false, spark-sql CLI prints the current database in prompt.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName",
+      "doc": "When true, columns will be looked up by name if labeled with a string and fallback to use position if not. When false, a grouped map Pandas UDF will assign columns from the returned Pandas DataFrame based on position, regardless of column label type. This configuration will be deprecated in future releases.",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "2.4",
+        "comment": "The config allows to switch to the behaviour before Spark 2.4 and will be removed in the future releases."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.exponentLiteralAsDecimal.enabled",
+      "doc": "When set to true, a literal with an exponent (e.g. 1E-30) would be parsed as Decimal rather than Double.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.extraOptionsBehavior.enabled",
+      "doc": "When true, the extra options will be ignored for DataFrameReader.table(). If set it to false, which is the default, Spark will check if the extra options have the same key, but the value is different with the table serde properties. If the check passes, the extra options will be merged with the serde properties as the scan options. Otherwise, an exception will be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.followThreeValuedLogicInArrayExists",
+      "doc": "When true, the ArrayExists will follow the three-valued boolean logic.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.fromDayTimeString.enabled",
+      "doc": "When true, the `from` bound is not taken into account in conversion of a day-time string to an interval, and the `to` bound is used to skip all interval units out of the specified range. If it is set to `false`, `ParseException` is thrown if the input does not match to the pattern defined by `from` and `to`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.groupingIdWithAppendedUserGroupBy",
+      "doc": "When true, grouping_id() returns values based on grouping set columns plus user-given group-by expressions order like Spark 3.2.0, 3.2.1, 3.2.2, and 3.3.0.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.histogramNumericPropagateInputType",
+      "doc": "The histogram_numeric function computes a histogram on numeric 'expr' using nb bins. The return value is an array of (x,y) pairs representing the centers of the histogram's bins. If this config is set to true, the output type of the 'x' field in the return value is propagated from the input value consumed in the aggregate function. Otherwise, 'x' always has double type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.inSubqueryNullability",
+      "doc": "When set to false, IN subquery nullability is correctly calculated based on both the left and right sides of the IN. When set to true, restores the legacy behavior that does not check the right side's nullability.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.inlineCTEInCommands",
+      "doc": "If true, always inline the CTE relations for the queries in commands. This is the legacy behavior which may produce incorrect results because Spark may evaluate a CTE relation more than once, even if it's nondeterministic.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.integerGroupingId",
+      "doc": "When true, grouping_id() returns int values instead of long values.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.interval.enabled",
+      "doc": "When set to true, Spark SQL uses the mixed legacy interval type `CalendarIntervalType` instead of the ANSI compliant interval types `YearMonthIntervalType` and `DayTimeIntervalType`. For instance, the date subtraction expression returns `CalendarIntervalType` when the SQL config is set to `true` otherwise an ANSI interval.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.javaCharsets",
+      "doc": "When set to true, the functions like `encode()` can use charsets from JDK while encoding or decoding string values. If it is false, such functions support only one of the charsets: 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.json.allowEmptyString.enabled",
+      "doc": "When set to true, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.json.enableDateTimeParsingFallback",
+      "doc": "When true, enable legacy date/time parsing fallback in JSON"
+    },
+    {
+      "key": "spark.sql.legacy.keepCommandOutputSchema",
+      "doc": "When true, Spark will keep the output schema of commands such as SHOW DATABASES unchanged.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.keepPartitionSpecAsStringLiteral",
+      "doc": "If it is set to true, `PARTITION(col=05)` is parsed as a string literal of its text representation, e.g., string '05', when the partition column is string type. Otherwise, it is always parsed as a numeric literal in the partition spec.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.literal.pickMinimumPrecision",
+      "doc": "When integral literal is used in decimal operations, pick a minimum precision required by the literal if this config is true, to make the resulting precision and/or scale smaller. This can reduce the possibility of precision lose and/or overflow.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.lpadRpadAlwaysReturnString",
+      "doc": "When set to false, when the first argument and the optional padding pattern is a byte sequence, the result is a BINARY value. The default padding pattern in this case is the zero byte. When set to true, it restores the legacy behavior of always returning string types even for binary inputs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mssqlserver.datetimeoffsetMapping.enabled",
+      "doc": "When true, DATETIMEOFFSET is mapped to StringType; otherwise, it is mapped to TimestampType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mssqlserver.numericMapping.enabled",
+      "doc": "When true, use legacy MsSqlServer TINYINT, SMALLINT and REAL type mapping.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mysql.bitArrayMapping.enabled",
+      "doc": "When true, use LongType to represent MySQL BIT(n>1); otherwise, use BinaryType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mysql.timestampNTZMapping.enabled",
+      "doc": "When true, TimestampNTZType and MySQL TIMESTAMP can be converted bidirectionally. For reading, MySQL TIMESTAMP is converted to TimestampNTZType when JDBC read option preferTimestampNTZ is true. For writing, TimestampNTZType is converted to MySQL TIMESTAMP; otherwise, DATETIME",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.negativeIndexInArrayInsert",
+      "doc": "When set to true, restores the legacy behavior of `array_insert` for negative indexes - 0-based: the function inserts new element before the last one for the index -1. For example, `array_insert(['a', 'b'], -1, 'x')` returns `['a', 'x', 'b']`. When set to false, the -1 index points out to the last element, and the given example produces `['a', 'b', 'x']`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.noCharPaddingInPredicate",
+      "doc": "When true, Spark will not apply char type padding for CHAR type columns in string comparison predicates, when 'spark.sql.readSideCharPadding' is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.notReserveProperties",
+      "doc": "When true, all database and table properties are not reserved and available for create/alter syntaxes. But please be aware that the reserved properties will be silently removed.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.nullInEmptyListBehavior",
+      "doc": "When set to true, restores the legacy incorrect behavior of IN expressions for NULL values IN an empty list (including IN subqueries and literal IN lists): `null IN (empty list)` should evaluate to false, but sometimes (not always) incorrectly evaluates to null in the legacy behavior."
+    },
+    {
+      "key": "spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv",
+      "doc": "When set to false, nulls are written as unquoted empty strings in CSV data source. If set to true, it restores the legacy behavior that nulls were written as quoted empty strings, `\"\"`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.oracle.timestampMapping.enabled",
+      "doc": "When true, TimestampType maps to TIMESTAMP in Oracle; otherwise, TIMESTAMP WITH LOCAL TIME ZONE.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parquet.datetimeRebaseModeInWrite",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.parquet.datetimeRebaseModeInWrite' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.parquet.int96RebaseModeInRead",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.parquet.int96RebaseModeInRead' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.parquet.int96RebaseModeInWrite",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.parquet.int96RebaseModeInWrite' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.parquet.nanosAsLong",
+      "doc": "When true, the Parquet's nanos precision timestamps are converted to SQL long values.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parseNullPartitionSpecAsStringLiteral",
+      "doc": "If it is set to true, `PARTITION(col=null)` is parsed as a string literal of its text representation, e.g., string 'null', when the partition column is string type. Otherwise, it is always parsed as a null literal in the partition spec.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parseQueryWithoutEof",
+      "doc": "When set to true, ParserInterface#parseQuery(...) is going to use base `query` grammar term without EOF resulting in some queries (like `SELECT 1 UNION SELECT 2`) to be parsed incorrectly - `UNION` will be treated as an alias, and the rest of SQL input will be thrown away.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parser.havingWithoutGroupByAsWhere",
+      "doc": "If it is set to true, the parser will treat HAVING without GROUP BY as a normal WHERE, which does not follow SQL standard.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.pathOptionBehavior.enabled",
+      "doc": "When true, \"path\" option is overwritten if one path parameter is passed to DataFrameReader.load(), DataFrameWriter.save(), DataStreamReader.load(), or DataStreamWriter.start(). Also, \"path\" option is added to the overall paths if multiple path parameters are passed to DataFrameReader.load()",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.percentileDiscCalculation",
+      "doc": "If true, the old bogus percentile_disc calculation is used. The old calculation incorrectly mapped the requested percentile to the sorted range of values in some cases and so returned incorrect results. Also, the new implementation is faster as it doesn't contain the interpolation logic that the old percentile_cont based one did.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.postgres.datetimeMapping.enabled",
+      "doc": "When true, TimestampType maps to TIMESTAMP WITHOUT TIME ZONE in PostgreSQL for writing; otherwise, TIMESTAMP WITH TIME ZONE. When true, TIMESTAMP WITH TIME ZONE can be converted to TimestampNTZType when JDBC read option preferTimestampNTZ is true; otherwise, converted to TimestampType regardless of preferTimestampNTZ.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.raiseErrorWithoutErrorClass",
+      "doc": "When set to true, restores the legacy behavior of `raise_error` and `assert_true` to not return the `[USER_RAISED_EXCEPTION]` prefix. For example, `raise_error('error!')` returns `error!` instead of `[USER_RAISED_EXCEPTION] Error!`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.readFileSourceTableCacheIgnoreOptions",
+      "doc": "When set to true, reading from file source table caches the first query plan and ignores subsequent changes in query options. Otherwise, query options will be applied to the cached plan and may produce different results.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.replaceDatabricksSparkAvro.enabled",
+      "doc": "If it is set to true, the data source provider com.databricks.spark.avro is mapped to the built-in but external Avro data source module for backward compatibility.",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use `.format(\"avro\")` in `DataFrameWriter` or `DataFrameReader` instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.respectNullabilityInTextDatasetConversion",
+      "doc": "When true, the nullability in the user-specified schema for `DataFrameReader.schema(schema).json(jsonDataset)` and `DataFrameReader.schema(schema).csv(csvDataset)` is respected. Otherwise, they are turned to a nullable schema forcibly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.scalarSubqueryAllowGroupByNonEqualityCorrelatedPredicate",
+      "doc": "When set to true, use incorrect legacy behavior for checking whether a scalar subquery with a group-by on correlated columns is allowed. See SPARK-48503",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.scalarSubqueryCountBugBehavior",
+      "doc": "When set to true, restores legacy behavior of potential incorrect count bug handling for scalar subqueries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.sessionInitWithConfigDefaults",
+      "doc": "Flag to revert to legacy behavior where a cloned SparkSession receives SparkConf defaults, dropping any overrides in its parent SparkSession.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.legacy.setCommandRejectsSparkCoreConfs",
+      "doc": "If it is set to true, SET command will fail when the key is registered as a SparkConf entry.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.setopsPrecedence.enabled",
+      "doc": "When set to true and the order of evaluation is not specified by parentheses, the set operations are performed from left to right as they appear in the query. When set to false and order of evaluation is not specified by parentheses, INTERSECT operations are performed before any UNION, EXCEPT and MINUS operations.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.sizeOfNull",
+      "doc": "If it is set to false, or spark.sql.ansi.enabled is true, then size of null returns null. Otherwise, it returns -1, which was inherited from Hive.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.skipTypeValidationOnAlterPartition",
+      "doc": "When true, skip validation for partition spec in ALTER PARTITION. E.g., `ALTER TABLE .. ADD PARTITION(p='a')` would work even the partition type is int. Besides, this config will also be used to skip type validation on partition spec when reading partitioned table. E.g., if the table partition spec is added without type validation, it might not be read correctly with the type validation. When false, the behavior follows spark.sql.storeAssignmentPolicy",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.statisticalAggregate",
+      "doc": "When set to true, statistical aggregate function returns Double.NaN if divide by zero occurred during expression evaluation, otherwise, it returns null. Before version 3.1.0, it returns NaN in divideByZero case by default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.storeAnalyzedPlanForView",
+      "doc": "When true, analyzed plan instead of SQL text will be stored when creating temporary view",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.timeParserPolicy",
+      "doc": "When LEGACY, java.text.SimpleDateFormat is used for formatting and parsing dates/timestamps in a locale-sensitive manner, which is the approach before Spark 3.0. When set to CORRECTED, classes from java.time.* packages are used for the same purpose. When set to EXCEPTION, RuntimeException is thrown when we will get different results. The default is CORRECTED.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.legacy.typeCoercion.datetimeToString.enabled",
+      "doc": "If it is set to true, date/timestamp will cast to string in binary comparisons with String when spark.sql.ansi.enabled is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useCurrentConfigsForView",
+      "doc": "When true, SQL Configs of the current active SparkSession instead of the captured ones will be applied during the parsing and analysis phases of the view resolution.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useV1Command",
+      "doc": "When true, Spark will use legacy V1 SQL commands.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.v1IdentifierNoCatalog",
+      "doc": "When set to false, the v1 identifier will include 'spark_catalog' as the catalog name if database is defined. When set to true, it restores the legacy behavior that does not include catalog name.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.viewSchemaBindingMode",
+      "doc": "Set to false to disable the WITH SCHEMA clause for view DDL and suppress the line in DESCRIBE EXTENDED and SHOW CREATE TABLE.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.viewSchemaCompensation",
+      "doc": "Set to false to revert default view schema binding mode from WITH SCHEMA COMPENSATION to WITH SCHEMA BINDING.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lightweightPlanChangeValidation",
+      "doc": "Similar to spark.sql.planChangeValidation, this validates plan changes and runs after every rule, however it is enabled by default and so it should be lightweight.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.limit.initialNumPartitions",
+      "doc": "Initial number of partitions to try when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as more jobs will be run",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.limit.scaleUpFactor",
+      "doc": "Minimal increase rate in number of partitions between attempts when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as more jobs will be run",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.mapKeyDedupPolicy",
+      "doc": "The policy to deduplicate map keys in builtin function: CreateMap, MapFromArrays, MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted at last takes precedence.",
+      "defaultValue": "EXCEPTION"
+    },
+    {
+      "key": "spark.sql.maven.additionalRemoteRepositories",
+      "doc": "A comma-delimited string config of the optional additional remote Maven mirror repositories. This is only used for downloading Hive jars in IsolatedClientLoader if the default Maven Central repo is unreachable.",
+      "defaultValue": "https://maven-central.storage-download.googleapis.com/maven2/"
+    },
+    {
+      "key": "spark.sql.maxConcurrentOutputFileWriters",
+      "doc": "Maximum number of output file writers to use concurrently. If number of writers needed reaches this limit, task will sort rest of output then writing them.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.maxMetadataStringLength",
+      "doc": "Maximum number of characters to output for a metadata string. e.g. file location in `DataSourceScanExec`, every value will be abbreviated if exceed length.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.maxPlanStringLength",
+      "doc": "Maximum number of characters to output for a plan string.  If the plan is longer, further output will be truncated.  The default setting always generates a full plan.  Set this to a lower value such as 8k if plan strings are taking up too much memory or are causing OutOfMemory errors in the driver or UI processes.",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.maxSinglePartitionBytes",
+      "doc": "The maximum number of bytes allowed for a single partition. Otherwise, The planner will introduce shuffle to improve parallelism.",
+      "defaultValue": "128m"
+    },
+    {
+      "key": "spark.sql.metadataCacheTTLSeconds",
+      "doc": "Time-to-live (TTL) value for the metadata caches: partition file metadata cache and session catalog cache. This configuration only has an effect when this value having a positive value (> 0). It also requires setting 'spark.sql.catalogImplementation' to `hive`, setting 'spark.sql.hive.filesourcePartitionFileCacheSize' > 0 and setting 'spark.sql.hive.manageFilesourcePartitions' to `true` to be applied to the partition file metadata cache.",
+      "defaultValue": "-1000ms",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.objectHashAggregate.sortBased.fallbackThreshold",
+      "doc": "In the case of ObjectHashAggregateExec, when the size of the in-memory hash map grows too large, we will fall back to sort-based aggregation. This option sets a row count threshold for the size of the hash map.",
+      "defaultValue": "128"
+    },
+    {
+      "key": "spark.sql.operatorPipeSyntaxEnabled",
+      "doc": "If true, enable operator pipe syntax for Apache Spark SQL. This uses the operator pipe marker |> to indicate separation between clauses of SQL in a manner that describes the sequence of steps that the query performs in a composable fashion.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizeNullAwareAntiJoin",
+      "doc": "When true, NULL-aware anti join execution will be planed into BroadcastHashJoinExec with flag isNullAwareAntiJoin enabled, optimized from O(M*N) calculation into O(M) calculation using Hash lookup instead of Looping lookup. Only support for singleColumn NAAJ for now.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.avoidCollapseUDFWithExpensiveExpr",
+      "doc": "Whether to avoid collapsing projections that would duplicate expensive expressions in UDFs.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.canChangeCachedPlanOutputPartitioning",
+      "doc": "Whether to forcibly enable some optimization rules that can change the output partitioning of a cached query when executing it for caching. If it is set to true, queries may need an extra shuffle to read the cached data. This configuration is disabled by default. The optimization rule enabled by this configuration is spark.sql.adaptive.applyFinalStageShuffleOptimizations.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.collapseProjectAlwaysInline",
+      "doc": "Whether to always collapse two adjacent projections and inline expressions even if it causes extra duplication.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateExistsSubqueryLegacyIncorrectCountHandling.enabled",
+      "doc": "If enabled, revert to legacy incorrect behavior for certain EXISTS/IN subqueries with COUNT or similar aggregates.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateInnerQuery.enabled",
+      "doc": "Decorrelate inner query by eliminating correlated references and build domain joins.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateJoinPredicate.enabled",
+      "doc": "Decorrelate scalar and lateral subqueries with correlated references in join predicates. This configuration is only effective when 'spark.sql.optimizer.decorrelateInnerQuery.enabled' is true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateLimit.enabled",
+      "doc": "Decorrelate subqueries with correlation under LIMIT.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateOffset.enabled",
+      "doc": "Decorrelate subqueries with correlation under LIMIT with OFFSET.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelatePredicateSubqueriesInJoinPredicate.enabled",
+      "doc": "Decorrelate predicate (in and exists) subqueries with correlated references in join predicates.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSetOps.enabled",
+      "doc": "Decorrelate subqueries with correlation under set operators.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSubqueryLegacyIncorrectCountHandling.enabled",
+      "doc": "If enabled, revert to legacy incorrect behavior for certain subqueries with COUNT or similar aggregates: see SPARK-43098.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSubqueryPreventConstantHoldingForCountBug.enabled",
+      "doc": "If enabled, prevents constant folding in subqueries that contain a COUNT-bug-susceptible Aggregate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateUnionOrSetOpUnderLimit.enabled",
+      "doc": "Decorrelate UNION or SET operation under LIMIT operator. If not enabled,revert to legacy incorrect behavior for certain subqueries with correlation underUNION/SET operator with a LIMIT operator above it.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.disableHints",
+      "doc": "When true, the optimizer will disable user-specified hints that are additional directives for better planning of a query.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.enabled",
+      "doc": "When true, we will generate predicate for partition column when it's used as join key",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio",
+      "doc": "When statistics are not available or configured not to be used, this config will be used as the fallback filter ratio for computing the data size of the partitioned table after dynamic partition pruning, in order to evaluate if it is worth adding an extra subquery as the pruning filter if broadcast reuse is not applicable.",
+      "defaultValue": "0.5"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly",
+      "doc": "When true, dynamic partition pruning will only apply when the broadcast exchange of a broadcast hash join operation can be reused as the dynamic pruning filter.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.useStats",
+      "doc": "When true, distinct count statistics will be used for computing the data size of the partitioned table after dynamic partition pruning, in order to evaluate if it is worth adding an extra subquery as the pruning filter if broadcast reuse is not applicable.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.enableCsvExpressionOptimization",
+      "doc": "Whether to optimize CSV expressions in SQL optimizer. It includes pruning unnecessary columns from from_csv.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.enableJsonExpressionOptimization",
+      "doc": "Whether to optimize JSON expressions in SQL optimizer. It includes pruning unnecessary columns from from_json, simplifying from_json + to_json, to_json + named_struct(from_json.col1, from_json.col2, ....).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.excludeSubqueryRefsFromRemoveRedundantAliases.enabled",
+      "doc": "When true, exclude the references from the subquery expressions (in, exists, etc.) while removing redundant aliases.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.excludedRules",
+      "doc": "Configures a list of rules to be disabled in the optimizer, in which the rules are specified by their rule names and separated by comma. It is not guaranteed that all the rules in this configuration will eventually be excluded, as some rules are necessary for correctness. The optimizer will log the rules that have indeed been excluded."
+    },
+    {
+      "key": "spark.sql.optimizer.expression.nestedPruning.enabled",
+      "doc": "Prune nested fields from expressions in an operator which are unnecessary in satisfying a query. Note that this optimization doesn't prune nested fields from physical data source scanning. For pruning nested fields from scanning, please use `spark.sql.optimizer.nestedSchemaPruning.enabled` config.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.expressionProjectionCandidateLimit",
+      "doc": "The maximum number of the candidate of output expressions whose alias are replaced. It can preserve the output partitioning and ordering. Negative value means disable this optimization.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.optimizer.inSetConversionThreshold",
+      "doc": "The threshold of set size for InSet conversion.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.optimizer.inSetSwitchThreshold",
+      "doc": "Configures the max set size in InSet for which Spark will generate code with switch statements. This is applicable only to bytes, shorts, ints, dates.",
+      "defaultValue": "400"
+    },
+    {
+      "key": "spark.sql.optimizer.maxIterations",
+      "doc": "The max number of iterations the optimizer runs.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.optimizer.metadataOnly",
+      "doc": "When true, enable the metadata-only query optimization that use the table's metadata to produce the partition columns instead of table scans. It applies when all the columns scanned are partition columns and the query has an aggregate operator that satisfies distinct semantics. By default the optimization is disabled, and deprecated as of Spark 3.0 since it may return incorrect results when the files are empty, see also SPARK-26709. It will be removed in the future releases. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Avoid to depend on this optimization to prevent a potential correctness issue. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources",
+      "doc": "A comma-separated list of data source short names or fully qualified data source implementation class names for which Spark tries to push down predicates for nested columns and/or names containing `dots` to data sources. This configuration is only effective with file-based data sources in DSv1. Currently, Parquet and ORC implement both optimizations. The other data sources don't support this feature yet. So the default value is 'parquet,orc'.",
+      "defaultValue": "parquet,orc"
+    },
+    {
+      "key": "spark.sql.optimizer.nestedSchemaPruning.enabled",
+      "doc": "Prune nested fields from a logical relation's output which are unnecessary in satisfying a query. This optimization allows columnar file format readers to avoid reading unnecessary nested column data. Currently Parquet and ORC are the data sources that implement this optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery",
+      "doc": "When true, the optimizer will inline subqueries with OneRowRelation as leaf nodes.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline",
+      "doc": "When true, the optimizer will always inline single row subqueries even if it causes extra duplication. It only takes effect when spark.sql.optimizer.optimizeOneRowRelationSubquery is set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeUncorrelatedInSubqueriesInJoinCondition.enabled",
+      "doc": "When true, optimize uncorrelated IN subqueries in join predicates by rewriting them to joins. This interacts with spark.sql.legacy.nullInEmptyListBehavior because it can rewrite IN predicates.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.batches",
+      "doc": "(Removed)",
+      "defaultValue": "",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.batches` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.level",
+      "doc": "(Removed)",
+      "defaultValue": "trace",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.level` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.rules",
+      "doc": "(Removed)",
+      "defaultValue": "",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.rules` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.plannedWrite.enabled",
+      "doc": "When set to true, Spark optimizer will add logical sort operators to V1 write commands if needed so that `FileFormatWriter` does not need to insert physical sorts.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.propagateDistinctKeys.enabled",
+      "doc": "When true, the query optimizer will propagate a set of distinct attributes from the current node and use it to optimize query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.pruneFiltersCanPruneStreamingSubplan",
+      "doc": "Allow PruneFilters to remove streaming subplans when we encounter a false filter. This flag is to restore prior buggy behavior for broken pipelines.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.pullHintsIntoSubqueries",
+      "doc": "Pull hints into subqueries in EliminateResolvedHint if enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.pullOutNestedDataOuterRefExpressions.enabled",
+      "doc": "Handle correlation over nested data extract expressions by pulling out the expression into the outer plan. This enables correlation on map attributes for example.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.replaceExceptWithFilter",
+      "doc": "When true, the apply function of the rule verifies whether the right node of the except operation is of type Filter or Project followed by Filter. If yes, the rule further verifies 1) Excluding the filter operations from the right (as well as the left node, if any) on the top, whether both the nodes evaluates to a same result. 2) The left and right nodes don't contain any SubqueryExpressions. 3) The output column names of the left node are distinct. If all the conditions are met, the rule will replace the except operation with a Filter by flipping the filter condition(s) of the right node.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold",
+      "doc": "Byte size threshold of the Bloom filter application side plan's aggregated scan size. Aggregated scan byte size of the Bloom filter application side needs to be over this value to inject a bloom filter.",
+      "defaultValue": "10GB"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold",
+      "doc": "Size threshold of the bloom filter creation side plan. Estimated size needs to be under this value to try to inject bloom filter.",
+      "defaultValue": "10MB"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.enabled",
+      "doc": "When true and if one side of a shuffle join has a selective predicate, we attempt to insert a bloom filter in the other side to reduce the amount of shuffle data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.expectedNumItems",
+      "doc": "The default number of expected items for the runtime bloomfilter",
+      "defaultValue": "1000000"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.maxNumBits",
+      "doc": "The max number of bits to use for the runtime bloom filter",
+      "defaultValue": "67108864"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.maxNumItems",
+      "doc": "The max allowed number of expected items for the runtime bloom filter",
+      "defaultValue": "4000000"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.numBits",
+      "doc": "The default number of bits to use for the runtime bloom filter",
+      "defaultValue": "8388608"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.rowLevelOperationGroupFilter.enabled",
+      "doc": "Enables runtime group filtering for group-based row-level operations. Data sources that replace groups of data (e.g. files, partitions) may prune entire groups using provided data source filters when planning a row-level operation scan. However, such filtering is limited as not all expressions can be converted into data source filters and some expressions can only be evaluated by Spark (e.g. subqueries). Since rewriting groups is expensive, Spark can execute a query at runtime to find what records match the condition of the row-level operation. The information about matching records will be passed back to the row-level operation scan, allowing data sources to discard groups that don't have to be rewritten.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtimeFilter.number.threshold",
+      "doc": "The total number of injected runtime filters (non-DPP) for a single query. This is to prevent driver OOMs with too many Bloom filters.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.optimizer.runtimeFilter.semiJoinReduction.enabled",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "This optimizer config is useless as runtime filter cannot be an IN subquery now."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.scalarSubqueryUseSingleJoin",
+      "doc": "When set to true, use LEFT_SINGLE join for correlated scalar subqueries where optimizer can't prove that only 1 row will be returned",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.serializer.nestedSchemaPruning.enabled",
+      "doc": "Prune nested fields from object serialization operator which are unnecessary in satisfying a query. This optimization allows object serializers to avoid executing unnecessary nested expressions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.windowGroupLimitThreshold",
+      "doc": "Threshold for triggering `InsertWindowGroupLimit`. 0 means the output results is empty. -1 means disabling the optimization.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.optimizer.wrapExistsInAggregateFunction",
+      "doc": "When true, the optimizer will wrap newly introduced `exists` attributes in an aggregate function to ensure that Aggregate nodes preserve semantic invariant that each variable among agg expressions appears either in grouping expressions or belongs to and aggregate function.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.aggregatePushdown",
+      "doc": "If true, aggregates will be pushed down to ORC for optimization. Support MIN, MAX and COUNT as aggregate expression. For MIN/MAX, support boolean, integer, float and date type. For COUNT, support all data types. If statistics is missing from any ORC file footer, exception would be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.orc.columnarReaderBatchSize",
+      "doc": "The number of rows to include in a orc vectorized reader batch. The number should be carefully chosen to minimize overhead and avoid OOMs in reading data.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.orc.columnarWriterBatchSize",
+      "doc": "The number of rows to include in a orc vectorized writer batch. The number should be carefully chosen to minimize overhead and avoid OOMs in writing data.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.orc.compression.codec",
+      "doc": "Sets the compression codec used when writing ORC files. If either `compression` or `orc.compress` is specified in the table-specific options/properties, the precedence would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`. Acceptable values include: none, uncompressed, snappy, zlib, lzo, zstd, lz4, brotli.",
+      "defaultValue": "zstd"
+    },
+    {
+      "key": "spark.sql.orc.enableNestedColumnVectorizedReader",
+      "doc": "Enables vectorized orc decoding for nested column.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.enableVectorizedReader",
+      "doc": "Enables vectorized orc decoding.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.filterPushdown",
+      "doc": "When true, enable filter pushdown for ORC files.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.impl",
+      "doc": "When native, use the native version of ORC support instead of the ORC library in Hive. It is 'hive' by default prior to Spark 2.4.",
+      "defaultValue": "native"
+    },
+    {
+      "key": "spark.sql.orc.mergeSchema",
+      "doc": "When true, the Orc data source merges schemas collected from all data files, otherwise the schema is picked from a random data file.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.orderByOrdinal",
+      "doc": "When true, the ordinal numbers are treated as the position in the select list. When false, the ordinal numbers in order/sort by clause are ignored.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orderingAwareLimitOffset",
+      "doc": "When set to true, a local sort will be inserted between GlobalLimitExec and single-partition ShuffleExchangeExec, if the underlying plan produces sorted data. This is because shuffle reader in Spark fetches shuffle blocks in a random order and can not preserve the data ordering, while LIMIT/OFFSET must preserve ordering.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.aggregatePushdown",
+      "doc": "If true, aggregates will be pushed down to Parquet for optimization. Support MIN, MAX and COUNT as aggregate expression. For MIN/MAX, support boolean, integer, float and date type. For COUNT, support all data types. If statistics is missing from any Parquet file footer, exception would be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.binaryAsString",
+      "doc": "Some other Parquet-producing systems, in particular Impala and older versions of Spark SQL, do not differentiate between binary data and strings when writing out the Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.columnarReaderBatchSize",
+      "doc": "The number of rows to include in a parquet vectorized reader batch. The number should be carefully chosen to minimize overhead and avoid OOMs in reading data.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.parquet.compression.codec",
+      "doc": "Sets the compression codec used when writing Parquet files. If either `compression` or `parquet.compression` is specified in the table-specific options/properties, the precedence would be `compression`, `parquet.compression`, `spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.parquet.datetimeRebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. When EXCEPTION, Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown. This config influences on reads of the following parquet logical types: DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: spark.sql.parquet.int96RebaseModeInRead.",
+      "defaultValue": "CORRECTED",
+      "alternatives": [
+        "spark.sql.legacy.parquet.datetimeRebaseModeInRead"
+      ]
+    },
+    {
+      "key": "spark.sql.parquet.datetimeRebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. When EXCEPTION, Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config influences on writes of the following parquet logical types: DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: spark.sql.parquet.int96RebaseModeInWrite.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.parquet.enableNestedColumnVectorizedReader",
+      "doc": "Enables vectorized Parquet decoding for nested columns (e.g., struct, list, map). Requires spark.sql.parquet.enableVectorizedReader to be enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.enableVectorizedReader",
+      "doc": "Enables vectorized parquet decoding.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.read.enabled",
+      "doc": "Field ID is a native field of the Parquet schema spec. When enabled, Parquet readers will use field IDs (if present) in the requested Spark schema to look up Parquet fields instead of using column names",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.read.ignoreMissing",
+      "doc": "When the Parquet file doesn't have any field IDs but the Spark read schema is using field IDs to read, we will silently return nulls when this flag is enabled, or error otherwise.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.write.enabled",
+      "doc": "Field ID is a native field of the Parquet schema spec. When enabled, Parquet writers will populate the field Id metadata (if present) in the Spark schema to the Parquet schema.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown",
+      "doc": "Enables Parquet filter push-down optimization when set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.date",
+      "doc": "If true, enables Parquet filter push-down optimization for Date. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.decimal",
+      "doc": "If true, enables Parquet filter push-down optimization for Decimal. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.string.startsWith",
+      "doc": "If true, enables Parquet filter push-down optimization for string startsWith function. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.stringPredicate",
+      "doc": "If true, enables Parquet filter push-down optimization for string predicate such as startsWith/endsWith/contains function. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "fallback": "spark.sql.parquet.filterPushdown.string.startsWith"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.timestamp",
+      "doc": "If true, enables Parquet filter push-down optimization for Timestamp. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.inferTimestampNTZ.enabled",
+      "doc": "When enabled, Parquet timestamp columns with annotation isAdjustedToUTC = false are inferred as TIMESTAMP_NTZ type during schema inference. Otherwise, all the Parquet timestamp columns are inferred as TIMESTAMP_LTZ types. Note that Spark writes the output schema into Parquet's footer metadata on file writing and leverages it on file reading. Thus this configuration only affects the schema inference on Parquet files which are not written by Spark.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.int64AsTimestampMillis",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "The config was deprecated since Spark 2.3. Use 'spark.sql.parquet.outputTimestampType' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.parquet.int96AsTimestamp",
+      "doc": "Some Parquet-producing systems, in particular Impala, store Timestamp into INT96. Spark would also store Timestamp as INT96 because we need to avoid precision lost of the nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.int96RebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase INT96 timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. When CORRECTED, Spark will not do rebase and read the timestamps as it is. When EXCEPTION, Spark will fail the reading if it sees ancient timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.parquet.int96RebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase INT96 timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. When CORRECTED, Spark will not do rebase and write the timestamps as it is. When EXCEPTION, Spark will fail the writing if it sees ancient timestamps that are ambiguous between the two calendars.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.parquet.int96TimestampConversion",
+      "doc": "This controls whether timestamp adjustments should be applied to INT96 data when converting to timestamps, for data written by Impala.  This is necessary because Impala stores INT96 data with a different timezone offset than Hive & Spark.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.mergeSchema",
+      "doc": "When true, the Parquet data source merges schemas collected from all data files, otherwise the schema is picked from the summary file or a random data file if no summary file is available.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.output.committer.class",
+      "doc": "The output committer class used by Parquet. The specified class needs to be a subclass of org.apache.hadoop.mapreduce.OutputCommitter. Typically, it's also a subclass of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata summaries will never be created, irrespective of the value of parquet.summary.metadata.level",
+      "defaultValue": "org.apache.parquet.hadoop.ParquetOutputCommitter"
+    },
+    {
+      "key": "spark.sql.parquet.outputTimestampType",
+      "doc": "Sets which Parquet timestamp type to use when Spark writes data to Parquet files. INT96 is a non-standard but commonly used timestamp type in Parquet. TIMESTAMP_MICROS is a standard timestamp type in Parquet, which stores number of microseconds from the Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which means Spark has to truncate the microsecond portion of its timestamp value.",
+      "defaultValue": "INT96"
+    },
+    {
+      "key": "spark.sql.parquet.pushdown.inFilterThreshold",
+      "doc": "For IN predicate, Parquet filter will push-down a set of OR clauses if its number of values not exceeds this threshold. Otherwise, Parquet filter will push-down a value greater than or equal to its minimum value and less than or equal to its maximum value. By setting this value to 0 this feature can be disabled. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.parquet.recordLevelFilter.enabled",
+      "doc": "If true, enables Parquet's native record-level filtering using the pushed down filters. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and the vectorized reader is not used. You can ensure the vectorized reader is not used by setting 'spark.sql.parquet.enableVectorizedReader' to false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.respectSummaryFiles",
+      "doc": "When true, we make assumption that all part-files of Parquet are consistent with summary files and we will ignore them when merging schema. Otherwise, if this is false, which is the default, we will merge all part-files. This should be considered as expert-only option, and shouldn't be enabled before knowing what it means exactly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.writeLegacyFormat",
+      "doc": "If true, data will be written in a way of Spark 1.4 and earlier. For example, decimal values will be written in Apache Parquet's fixed-length byte array format, which other systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet will be used. For example, decimals will be written in int-based format. If Parquet output is intended for use with systems that do not support this newer format, set to true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parser.eagerEvalOfUnresolvedInlineTable",
+      "doc": "Controls whether we optimize the ASTree that gets generated when parsing VALUES lists (UnresolvedInlineTable) by eagerly evaluating it in the AST Builder.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parser.escapedStringLiterals",
+      "doc": "When true, string literals (including regex patterns) remain escaped in our SQL parser. The default is false since Spark 2.0. Setting it to true can restore the behavior prior to Spark 2.0.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Use raw string literals with the `r` prefix instead. "
+      }
+    },
+    {
+      "key": "spark.sql.parser.quotedRegexColumnNames",
+      "doc": "When true, quoted Identifiers (using backticks) in SELECT statement are interpreted as regular expressions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pivotMaxValues",
+      "doc": "When doing a pivot without specifying values for the pivot column this is the maximum number of (distinct) values that will be collected without error.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.planChangeLog.batches",
+      "doc": "Configures a list of batches for logging plan changes, in which the batches are specified by their batch names and separated by comma."
+    },
+    {
+      "key": "spark.sql.planChangeLog.level",
+      "doc": "Configures the log level for logging the change from the original plan to the new plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'trace'.",
+      "defaultValue": "trace"
+    },
+    {
+      "key": "spark.sql.planChangeLog.rules",
+      "doc": "Configures a list of rules for logging plan changes, in which the rules are specified by their rule names and separated by comma."
+    },
+    {
+      "key": "spark.sql.planChangeValidation",
+      "doc": "If true, Spark will validate all the plan changes made by analyzer/optimizer and other catalyst rules, to make sure every rule returns a valid plan",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.planner.pythonExecution.memory",
+      "doc": "Specifies the memory allocation for executing Python code in Spark driver, in MiB. When set, it caps the memory for Python execution to the specified amount. If not set, Spark will not limit Python's memory usage and it is up to the application to avoid exceeding the overhead memory space shared with other non-JVM processes.\nNote: Windows does not support resource limiting and actual resource is not limited on MacOS."
+    },
+    {
+      "key": "spark.sql.preserveCharVarcharTypeInfo",
+      "doc": "When true, Spark does not replace CHAR/VARCHAR types the STRING type, which is the default behavior of Spark 3.0 and earlier versions. This means the length checks for CHAR/VARCHAR types is enforced and CHAR type is also properly padded.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.inferNestedDictAsStruct.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the nested dict as a map by default. When it set to true, it infers the nested dict as a struct.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.jvmStacktrace.enabled",
+      "doc": "When true, it shows the JVM stacktrace in the user-facing PySpark exception together with Python stacktrace. By default, it is disabled to hide JVM stacktrace and shows a Python-friendly exception only. Note that this is independent from log level settings.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the element type of an array from all values in the array by default. If this config is set to true, it restores the legacy behavior of only inferring the type from the first array element.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.legacy.inferMapTypeFromFirstPair.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the key/value types of a map from all pairs in the map by default. If this config is set to true, it restores the legacy behavior of only inferring the type from the first non-null pair.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.plotting.max_rows",
+      "doc": "The visual limit on plots. If set to 1000 for top-n-based plots (pie, bar, barh), the first 1000 data points will be used for plotting. For sampled-based plots (scatter, area, line), 1000 data points will be randomly sampled.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.pyspark.udf.profiler",
+      "doc": "Configure the Python/Pandas UDF profiler by enabling or disabling it with the option to choose between \"perf\" and \"memory\" types, or unsetting the config disables the profiler. This is disabled by default."
+    },
+    {
+      "key": "spark.sql.queryExecutionListeners",
+      "doc": "List of class names implementing QueryExecutionListener that will be automatically added to newly created sessions. The classes should have either a no-arg constructor, or a constructor that expects a SparkConf argument.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.readSideCharPadding",
+      "doc": "When true, Spark applies string padding when reading CHAR type columns/fields, in addition to the write-side padding. This config is true by default to better enforce CHAR type semantic in cases such as external tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.redaction.options.regex",
+      "doc": "Regex to decide which keys in a Spark SQL command's options map contain sensitive information. The values of options whose names that match this regex will be redacted in the explain output. This redaction is applied on top of the global redaction configuration defined by spark.redaction.regex.",
+      "defaultValue": "(?i)url"
+    },
+    {
+      "key": "spark.sql.redaction.string.regex",
+      "doc": "Regex to decide which parts of strings produced by Spark contain sensitive information. When this regex matches a string part, that string part is replaced by a dummy value. This is currently used to redact the output of SQL explain commands. When this conf is not set, the value from `spark.redaction.string.regex` is used.",
+      "fallback": "spark.redaction.string.regex"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.enabled",
+      "doc": "Enables eager evaluation or not. When true, the top K rows of Dataset will be displayed if and only if the REPL supports the eager evaluation. Currently, the eager evaluation is supported in PySpark and SparkR. In PySpark, for the notebooks like Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned outputs are showed similar to R data.frame would.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.maxNumRows",
+      "doc": "The max number of rows that are returned by eager evaluation. This only takes effect when spark.sql.repl.eagerEval.enabled is set to true. The valid range of this config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).",
+      "defaultValue": "20"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.truncate",
+      "doc": "The max number of characters for each cell that is returned by eager evaluation. This only takes effect when spark.sql.repl.eagerEval.enabled is set to true.",
+      "defaultValue": "20"
+    },
+    {
+      "key": "spark.sql.requireAllClusterKeysForCoPartition",
+      "doc": "When true, the planner requires all the clustering keys as the hash partition keys of the children, to eliminate the shuffles for the operator that needs its children to be co-partitioned, such as JOIN node. This is to avoid data skews which can lead to significant performance regression if shuffles are eliminated.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.requireAllClusterKeysForDistribution",
+      "doc": "When true, the planner requires all the clustering keys as the partition keys (with same ordering) of the children, to eliminate the shuffle for the operator that requires its children be clustered distributed, such as AGGREGATE and WINDOW node. This is to avoid data skews which can lead to significant performance regression if shuffle is eliminated.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.resultQueryStage.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to execute ResultQueryStageExec in AQE",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.retainGroupColumns",
+      "doc": "",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.runCollationTypeCastsBeforeAliasAssignment.enabled",
+      "doc": "When set to true, rules like ResolveAliases or ResolveAggregateFunctions will run CollationTypeCasts before alias assignment. This is necessary for correct alias generation.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.runSQLOnFiles",
+      "doc": "When true, we could use `datasource`.`path` as table in SQL query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.scriptTransformation.exitTimeoutInSeconds",
+      "doc": "Timeout for executor to wait for the termination of transformation script when EOF.",
+      "defaultValue": "10000ms"
+    },
+    {
+      "key": "spark.sql.scripting.enabled",
+      "doc": "SQL Scripting feature is under development and its use should be done under this feature flag. SQL Scripting enables users to write procedural SQL including control flow and error handling.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.selfJoinAutoResolveAmbiguity",
+      "doc": "",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.session.localRelationCacheThreshold",
+      "doc": "The threshold for the size in bytes of local relations to be cached at the driver side after serialization.",
+      "defaultValue": "67108864"
+    },
+    {
+      "key": "spark.sql.session.timeZone",
+      "doc": "The ID of session local timezone in the format of either region-based zone IDs or zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format '(+|-)HH', '(+|-)HH:mm' or '(+|-)HH:mm:ss', e.g '-08', '+01:00' or '-13:33:33'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous.",
+      "defaultValue": null
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.in.memory.threshold",
+      "doc": "Threshold for number of windows guaranteed to be held in memory by the session window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.shuffle.partitions",
+      "doc": "The default number of partitions to use when shuffling data for joins or aggregations. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.shuffleDependency.fileCleanup.enabled",
+      "doc": "When enabled, shuffle files will be cleaned up at the end of Spark Connect SQL executions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.shuffleDependency.skipMigration.enabled",
+      "doc": "When enabled, shuffle dependencies for a Spark Connect SQL execution are marked at the end of the execution, and they will not be migrated during decommissions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.shuffleExchange.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism for doing preparation of shuffle exchange, which includes subquery execution, file listing, etc.",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.shuffledHashJoinFactor",
+      "doc": "The shuffle hash join can be selected if the data size of small side multiplied by this factor is still smaller than the large side.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.sort.enableRadixSort",
+      "doc": "When true, enable use of radix sort when possible. Radix sort is much faster but requires additional memory to be reserved up-front. The memory overhead may be significant when sorting very small rows (up to 50% more in this case).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the sort merge join operator",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by sort merge join operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.sources.binaryFile.maxLength",
+      "doc": "The max length of a file that can be read by the binary file data source. Spark will fail fast and not attempt to read the file if its length exceeds this value. The theoretical max is Int.MaxValue, though VMs might implement a smaller max.",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.autoBucketedScan.enabled",
+      "doc": "When true, decide whether to do bucketed scan on input tables based on query plan automatically. Do not use bucketed scan if 1. query does not have operators to utilize bucketing (e.g. join, group-by, etc), or 2. there's an exchange operator between these operators and table scan. Note when 'spark.sql.sources.bucketing.enabled' is set to false, this configuration does not take any effect.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.enabled",
+      "doc": "When false, we will treat bucketed table as normal table",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.maxBuckets",
+      "doc": "The maximum number of buckets allowed.",
+      "defaultValue": "100000"
+    },
+    {
+      "key": "spark.sql.sources.commitProtocolClass",
+      "doc": "",
+      "defaultValue": "org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol"
+    },
+    {
+      "key": "spark.sql.sources.default",
+      "doc": "The default data source to use in input/output.",
+      "defaultValue": "parquet"
+    },
+    {
+      "key": "spark.sql.sources.disabledJdbcConnProviderList",
+      "doc": "Configures a list of JDBC connection providers, which are disabled. The list contains the name of the JDBC connection providers separated by comma.",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.sources.fileCompressionFactor",
+      "doc": "When estimating the output data size of a table scan, multiply the file size with this factor as the estimated data size, in case the data is compressed in the file and lead to a heavily underestimated result.",
+      "defaultValue": "1.0"
+    },
+    {
+      "key": "spark.sql.sources.ignoreDataLocality",
+      "doc": "If true, Spark will not fetch the block locations for each file on listing files. This speeds up file listing, but the scheduler cannot schedule tasks to take advantage of data locality. It can be particularly useful if data is read from a remote cluster so the scheduler could never take advantage of locality anyway.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.outputCommitterClass",
+      "doc": ""
+    },
+    {
+      "key": "spark.sql.sources.parallelPartitionDiscovery.parallelism",
+      "doc": "The number of parallelism to list a collection of path recursively, Set the number to prevent file listing from generating too many tasks.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.sources.parallelPartitionDiscovery.threshold",
+      "doc": "The maximum number of paths allowed for listing files at driver side. If the number of detected paths exceeds this value during partition discovery, it tries to list the files with another Spark distributed job. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "32"
+    },
+    {
+      "key": "spark.sql.sources.partitionColumnTypeInference.enabled",
+      "doc": "When true, automatically infer the data types for partitioned columns.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.partitionOverwriteMode",
+      "doc": "When INSERT OVERWRITE a partitioned data source table, we currently support 2 modes: static and dynamic. In static mode, Spark deletes all the partitions that match the partition specification(e.g. PARTITION(a=1,b)) in the INSERT statement, before overwriting. In dynamic mode, Spark doesn't delete partitions ahead, and only overwrite those partitions that have data written into it at runtime. By default we use static mode to keep the same behavior of Spark prior to 2.3. Note that this config doesn't affect Hive serde tables, as they are always overwritten with dynamic mode. This can also be set as an output option for a data source using key partitionOverwriteMode (which takes precedence over this setting), e.g. dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path).",
+      "defaultValue": "STATIC"
+    },
+    {
+      "key": "spark.sql.sources.schemaStringLengthThreshold",
+      "doc": "The maximum length allowed in a single cell when storing additional schema information in Hive's metastore.",
+      "defaultValue": "4000",
+      "isStatic": true,
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use 'spark.sql.hive.tablePropertyLengthThreshold' instead."
+      }
+    },
+    {
+      "key": "spark.sql.sources.useListFilesFileSystemList",
+      "doc": "A comma-separated list of file system schemes to use FileSystem.listFiles API for a single root path listing",
+      "defaultValue": "s3a"
+    },
+    {
+      "key": "spark.sql.sources.useV1SourceList",
+      "doc": "A comma-separated list of data source short names or fully qualified data source implementation class names for which Data Source V2 code path is disabled. These data sources will fallback to Data Source V1 code path.",
+      "defaultValue": "avro,csv,json,kafka,orc,parquet,text"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.allowCompatibleTransforms.enabled",
+      "doc": "Whether to allow storage-partition join in the case where the partition transforms are compatible but not identical.  This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled and spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled to be disabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.allowJoinKeysSubsetOfPartitionKeys.enabled",
+      "doc": "Whether to allow storage-partition join in the case where join keys are a subset of the partition keys of the source tables. At planning time, Spark will group the partitions by only those keys that are in the join keys. This is currently enabled only if spark.sql.requireAllClusterKeysForDistribution is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.enabled",
+      "doc": "Similar to spark.sql.sources.bucketing.enabled, this config is used to enable bucketing for V2 data sources. When turned on, Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and will try to avoid shuffle if necessary.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled",
+      "doc": "During a storage-partitioned join, whether to allow input partitions to be partially clustered, when both sides of the join are of KeyGroupedPartitioning. At planning time, Spark will pick the side with less data size based on table statistics, group and replicate them to match the other side. This is an optimization on skew join and can help to reduce data skewness when certain partitions are assigned large amount of data. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.partition.filter.enabled",
+      "doc": "Whether to filter partitions when running storage-partition join. When enabled, partitions without matches on the other side can be omitted for scanning, if allowed by the join type. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.pushPartValues.enabled",
+      "doc": "Whether to pushdown common partition values when spark.sql.sources.v2.bucketing.enabled is enabled. When turned on, if both sides of a join are of KeyGroupedPartitioning and if they share compatible partition keys, even if they don't have the exact same partition values, Spark will calculate a superset of partition values and pushdown that info to scan nodes, which will use empty partitions for the missing partition values on either side. This could help to eliminate unnecessary shuffles",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.shuffle.enabled",
+      "doc": "During a storage-partitioned join, whether to allow to shuffle only one side. When only one side is KeyGroupedPartitioning, if the conditions are met, spark will only shuffle the other side. This optimization will reduce the amount of data that needs to be shuffle. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.sorting.enabled",
+      "doc": "When turned on, Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and will try to avoid a shuffle if possible when sorting by those columns. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.validatePartitionColumns",
+      "doc": "When this option is set to true, partition column values will be validated with user-specified schema. If the validation fails, a runtime exception is thrown. When this option is set to false, the partition column value will be converted to null if it can not be casted to corresponding user-specified schema.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.stableDerivedColumnAlias.enabled",
+      "doc": "Enable deriving of stable column aliases from the lexer tree instead of parse tree and form them via pretty SQL print.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.stackTracesInDataFrameContext",
+      "doc": "The number of non-Spark stack traces in the captured DataFrame query context.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.statistics.fallBackToHdfs",
+      "doc": "When true, it will fall back to HDFS if the table statistics are not available from table metadata. This is useful in determining if a table is small enough to use broadcast joins. This flag is effective only for non-partitioned Hive tables. For non-partitioned data source tables, it will be automatically recalculated if table statistics are not available. For partitioned data source and partitioned Hive tables, It is 'spark.sql.defaultSizeInBytes' if table statistics are not available.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.histogram.enabled",
+      "doc": "Generates histograms when computing column statistics if enabled. Histograms can provide better estimation accuracy. Currently, Spark only supports equi-height histogram. Note that collecting histograms takes extra cost. For example, collecting column statistics usually takes only one table scan, but generating equi-height histogram will cause an extra table scan.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.histogram.numBins",
+      "doc": "The number of bins when generating histograms.",
+      "defaultValue": "254"
+    },
+    {
+      "key": "spark.sql.statistics.ndv.maxError",
+      "doc": "The maximum relative standard deviation allowed in HyperLogLog++ algorithm when generating column level statistics.",
+      "defaultValue": "0.05"
+    },
+    {
+      "key": "spark.sql.statistics.parallelFileListingInStatsComputation.enabled",
+      "doc": "When true, SQL commands use parallel file listing, as opposed to single thread listing. This usually speeds up commands that need to list many directories.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.statistics.percentile.accuracy",
+      "doc": "Accuracy of percentile approximation when generating equi-height histograms. Larger value means better accuracy. The relative error can be deduced by 1.0 / PERCENTILE_ACCURACY.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.statistics.size.autoUpdate.enabled",
+      "doc": "Enables automatic update for table size once table's data is changed. Note that if the total number of files of the table is very large, this can be expensive and slow down data change commands.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.updatePartitionStatsInAnalyzeTable.enabled",
+      "doc": "When this config is enabled, Spark will also update partition statistics in analyze table command (i.e., ANALYZE TABLE .. COMPUTE STATISTICS [NOSCAN]). Note the command will also become more expensive. When this config is disabled, Spark will only update table level statistics.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.storeAssignmentPolicy",
+      "doc": "When inserting a value into a column with different data type, Spark will perform type coercion. Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy, Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL. It disallows certain unreasonable type conversions such as converting `string` to `int` or `double` to `boolean`. With legacy policy, Spark allows the type coercion as long as it is a valid `Cast`, which is very loose. e.g. converting `string` to `int` or `double` to `boolean` is allowed. It is also the only behavior in Spark 2.x and it is compatible with Hive. With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is not allowed.",
+      "defaultValue": "ANSI"
+    },
+    {
+      "key": "spark.sql.streaming.aggregation.stateFormatVersion",
+      "doc": "State format version used by streaming aggregation operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.asyncLogPurge.enabled",
+      "doc": "When true, purging the offset log and commit log of old entries will be done asynchronously.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.escapedPathCheck.enabled",
+      "doc": "Whether to detect a streaming query may pick up an incorrect checkpoint path due to SPARK-26824.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.renamedFileCheck.enabled",
+      "doc": "When true, Spark will validate if renamed checkpoint file exists.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.checkpointLocation",
+      "doc": "The default location for storing checkpoint data for streaming queries."
+    },
+    {
+      "key": "spark.sql.streaming.commitProtocolClass",
+      "doc": "",
+      "defaultValue": "org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.epochBacklogQueueSize",
+      "doc": "The max number of entries to be stored in queue to wait for late epochs. If this parameter is exceeded by the size of the queue, stream will stop with an error.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.executorPollIntervalMs",
+      "doc": "The interval at which continuous execution readers will poll to check whether the epoch has advanced on the driver.",
+      "defaultValue": "100ms"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.executorQueueSize",
+      "doc": "The size (measured in number of rows) of the queue used in continuous execution to buffer the results of a ContinuousDataReader.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.streaming.disabledV2MicroBatchReaders",
+      "doc": "A comma-separated list of fully qualified data source register class names for which MicroBatchReadSupport is disabled. Reads from these sources will fall back to the V1 Sources.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.streaming.disabledV2Writers",
+      "doc": "A comma-separated list of fully qualified data source register class names for which StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.cleanupDelay",
+      "doc": "How long that a file is guaranteed to be visible for all readers.",
+      "defaultValue": "600000ms"
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.compactInterval",
+      "doc": "Number of log files after which all the previous files are compacted into the next log file.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.deletion",
+      "doc": "Whether to delete the expired log files in file stream sink.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.cleaner.numThreads",
+      "doc": "Number of threads used in the file source completed file cleaner.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.cleanupDelay",
+      "doc": "How long in milliseconds a file is guaranteed to be visible for all readers.",
+      "defaultValue": "600000ms"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.compactInterval",
+      "doc": "Number of log files after which all the previous files are compacted into the next log file.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.deletion",
+      "doc": "Whether to delete the expired log files in file stream source.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.schema.forceNullable",
+      "doc": "When true, force the schema of streaming file source to be nullable (including all the fields). Otherwise, the schema might not be compatible with actual data, which leads to corruptions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileStreamSink.ignoreMetadata",
+      "doc": "If this is enabled, when Spark reads from the results of a streaming query written by `FileStreamSink`, Spark will ignore the metadata log and treat it as normal path to read, e.g. listing files using HDFS APIs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.flatMapGroupsWithState.skipEmittingInitialStateKeys",
+      "doc": "When true, the flatMapGroupsWithState operation in a streaming query will not emit results for the initial state keys of each group.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion",
+      "doc": "State format version used by flatMapGroupsWithState operation in a streaming query",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.forceDeleteTempCheckpointLocation",
+      "doc": "When true, enable temporary checkpoint locations force delete.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.join.stateFormatVersion",
+      "doc": "State format version used by streaming join operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.kafka.useDeprecatedOffsetFetching",
+      "doc": "When true, the deprecated Consumer based offset fetching used which could cause infinite wait in Spark queries. Such cases query restart is the only workaround. For further details please see Offset Fetching chapter of Structured Streaming Kafka Integration Guide.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.maxBatchesToRetainInMemory",
+      "doc": "The maximum number of batches which will be retained in memory to avoid loading from files. The value adjusts a trade-off between memory usage vs cache miss: '2' covers both success and direct failure cases, '1' covers only success case, and '0' covers extreme case - disable cache to maximize memory size of executors.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.metadataCache.enabled",
+      "doc": "Whether the streaming HDFSMetadataLog caches the metadata of the latest two batches.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.metricsEnabled",
+      "doc": "Whether Dropwizard/Codahale metrics will be reported for active streaming queries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.minBatchesToRetain",
+      "doc": "The minimum number of batches that must be retained and made recoverable.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.streaming.multipleWatermarkPolicy",
+      "doc": "Policy to calculate the global watermark value when there are multiple watermark operators in a streaming query. The default value is 'min' which chooses the minimum watermark reported across multiple operators. Other alternative value is 'max' which chooses the maximum across multiple operators. Note: This configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "min"
+    },
+    {
+      "key": "spark.sql.streaming.noDataMicroBatches.enabled",
+      "doc": "Whether streaming micro-batch engine will execute batches without data for eager state management for stateful streaming queries.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.noDataProgressEventInterval",
+      "doc": "How long to wait before providing query idle event when there is no data",
+      "defaultValue": "10000ms"
+    },
+    {
+      "key": "spark.sql.streaming.numRecentProgressUpdates",
+      "doc": "The number of progress updates to retain for a streaming query",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.streaming.optimizeOneRowPlan.enabled",
+      "doc": "When true, enable OptimizeOneRowPlan rule for the case where the child is a streaming Dataset. This is a fallback flag to revert the 'incorrect' behavior, hence this configuration must not be used without understanding in depth. Use this only to quickly recover failure in existing query!",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.pollingDelay",
+      "doc": "How long to delay polling new data when no data is available",
+      "defaultValue": "10ms"
+    },
+    {
+      "key": "spark.sql.streaming.ratioExtraSpaceAllowedInCheckpoint",
+      "doc": "The ratio of extra space allowed for batch deletion of files when maintenance isinvoked. When value > 0, it optimizes the cost of discovering and deleting old checkpoint versions. The minimum number of stale versions we retain in checkpoint location for batch deletion is calculated by minBatchesToRetain * ratioExtraSpaceAllowedInCheckpoint.",
+      "defaultValue": "0.3"
+    },
+    {
+      "key": "spark.sql.streaming.schemaInference",
+      "doc": "Whether file-based streaming sources will infer its own schema",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.sessionWindow.merge.sessions.in.local.partition",
+      "doc": "When true, streaming session window sorts and merge sessions in local partition prior to shuffle. This is to reduce the rows to shuffle, but only beneficial when there're lots of rows in a batch being assigned to same sessions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.sessionWindow.stateFormatVersion",
+      "doc": "State format version used by streaming session window in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.checkpointFormatVersion",
+      "doc": "The version of the approach of doing state store checkpoint",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.compression.codec",
+      "doc": "The codec used to compress delta and snapshot files generated by StateStore. By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify the codec. Default codec is lz4.",
+      "defaultValue": "lz4"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.encodingFormat",
+      "doc": "The encoding format used for stateful operators to store information in the state store",
+      "defaultValue": "unsaferow"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.formatValidation.enabled",
+      "doc": "When true, check if the data from state store is valid or not when running streaming queries. This can happen if the state store format has been changed. Note, the feature is only effective in the build-in HDFS state store provider now.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maintenanceInterval",
+      "doc": "The interval in milliseconds between triggering maintenance tasks in StateStore. The maintenance task executes background maintenance task in all the loaded store providers if they are still the active instances according to the coordinator. If not, inactive instances of store providers will be closed.",
+      "defaultValue": "60000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maxNumStateSchemaFiles",
+      "doc": "The maximum number of StateSchemaV3 files allowed per operator",
+      "defaultValue": "128"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.minDeltasForSnapshot",
+      "doc": "Minimum number of state store delta files that needs to be generated before they consolidated into snapshots.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.numStateStoreMaintenanceThreads",
+      "doc": "Number of threads in the thread pool that perform clean up and snapshotting tasks for stateful streaming queries. The default value is the number of cores * 0.25 so that this thread pool doesn't take too many resources away from the query and affect performance.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.providerClass",
+      "doc": "The class used to manage state data in stateful streaming queries. This class must be a subclass of StateStoreProvider, and must have a zero-arg constructor. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.rocksdb.formatVersion",
+      "doc": "Set the RocksDB format version. This will be stored in the checkpoint when starting a streaming query. The checkpoint will use this RocksDB format version in the entire lifetime of the query.",
+      "defaultValue": "5"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.skipNullsForStreamStreamJoins.enabled",
+      "doc": "When true, this config will skip null values in hash based stream-stream joins. The number of skipped null values will be shown as custom metric of stream join operator. If the streaming query was started with Spark 3.5 or above, please exercise caution before enabling this config since it may hide potential data loss/corruption issues.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.stateSchemaCheck",
+      "doc": "When true, Spark will validate the state schema against schema on existing state and fail query if it's incompatible.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.valueStateSchemaEvolutionThreshold",
+      "doc": "The maximum number of value state schema evolutions allowed per column family",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.allowMultiple",
+      "doc": "When true, multiple stateful operators are allowed to be present in a streaming pipeline. The support for multiple stateful operators introduces a minor (semantically correct) change in respect to late record filtering - late records are detected and filtered in respect to the watermark from the previous microbatch instead of the current one. This is a behavior change for Spark streaming pipelines and we allow users to revert to the previous behavior of late record filtering (late records are detected and filtered by comparing with the current microbatch watermark) by setting the flag value to false. In this mode, only a single stateful operator will be allowed in a streaming pipeline.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.checkCorrectness.enabled",
+      "doc": "When true, the stateful operators for streaming query will be checked for possible correctness issue due to global watermark. The correctness issue comes from queries containing stateful operation which can emit rows older than the current watermark plus allowed late record delay, which are \"late rows\" in downstream stateful operations and these rows can be discarded. Please refer the programming guide doc for more details. Once the issue is detected, Spark will throw analysis exception. When this config is disabled, Spark will just print warning message for users. Prior to Spark 3.1.0, the behavior is disabling this config.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.useStrictDistribution",
+      "doc": "The purpose of this config is only compatibility; DO NOT MANUALLY CHANGE THIS!!! When true, the stateful operator for streaming query will use StatefulOpClusteredDistribution which guarantees stable state partitioning as long as the operator provides consistent grouping keys across the lifetime of query. When false, the stateful operator for streaming query will use ClusteredDistribution which is not sufficient to guarantee stable state partitioning despite the operator provides consistent grouping keys across the lifetime of query. This config will be set to true for new streaming queries to guarantee stable state partitioning, and set to false for existing streaming queries to not break queries which are restored from existing checkpoints. Please refer SPARK-38204 for details.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stopActiveRunOnRestart",
+      "doc": "Running multiple runs of the same streaming query concurrently is not supported. If we find a concurrent active run for a streaming query (in the same or different SparkSessions on the same cluster) and this flag is true, we will stop the old streaming query run to start the new one.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stopTimeout",
+      "doc": "How long to wait in milliseconds for the streaming execution thread to stop when calling the streaming query's stop() method. 0 or negative values wait indefinitely.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.streaming.streamingQueryListeners",
+      "doc": "List of class names implementing StreamingQueryListener that will be automatically added to newly created sessions. The classes should have either a no-arg constructor, or a constructor that expects a SparkConf argument.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.transformWithState.stateSchemaVersion",
+      "doc": "The version of the state schema used by the transformWithState operator",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.streaming.triggerAvailableNowWrapper.enabled",
+      "doc": "Whether to use the wrapper implementation of Trigger.AvailableNow if the source does not support Trigger.AvailableNow. Enabling this allows the benefits of Trigger.AvailableNow with sources which don't support it, but some sources may show unexpected behavior including duplication, data loss, etc. So use with extreme care! The ideal direction is to persuade developers of source(s) to support Trigger.AvailableNow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.ui.enabled",
+      "doc": "Whether to run the Structured Streaming Web UI for the Spark application when the Spark Web UI is enabled.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.enabledCustomMetricList",
+      "doc": "Configures a list of custom metrics on Structured Streaming UI, which are enabled. The list contains the name of the custom metrics separated by comma. In aggregation only sum used. The list of supported custom metrics is state store provider specific and it can be found out for example from query progress log entry.",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.retainedProgressUpdates",
+      "doc": "The number of progress updates to retain for a streaming query for Structured Streaming UI.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.retainedQueries",
+      "doc": "The number of inactive queries to retain for Structured Streaming UI.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.unsupportedOperationCheck",
+      "doc": "When true, the logical plan for streaming query will be checked for unsupported operations.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.cache.maxEntries",
+      "doc": "The maximum entries of the cache used for interpreted subexpression elimination.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.enabled",
+      "doc": "When true, common subexpressions will be eliminated.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.skipForShortcutExpr",
+      "doc": "When true, shortcut eliminate subexpression with `AND`, `OR`. The subexpression may not need to eval even if it appears more than once. e.g., `if(or(a, and(b, b)))`, the expression `b` would be skipped if `a` is true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.subquery.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to execute the subquery.",
+      "defaultValue": "16",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.thriftServer.incrementalCollect",
+      "doc": "When true, enable incremental collection for execution in Thrift Server.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.thriftServer.interruptOnCancel",
+      "doc": "When true, all running tasks will be interrupted if one cancels a query. When false, all running tasks will remain until finished.",
+      "fallback": "spark.sql.execution.interruptOnCancel"
+    },
+    {
+      "key": "spark.sql.thriftServer.queryTimeout",
+      "doc": "Set a query duration timeout in seconds in Thrift Server. If the timeout is set to a positive value, a running query will be cancelled automatically when the timeout is exceeded, otherwise the query continues to run till completion. If timeout values are set for each statement via `java.sql.Statement.setQueryTimeout` and they are smaller than this configuration value, they take precedence. If you set this timeout and prefer to cancel the queries right away without waiting task to finish, consider enabling spark.sql.thriftServer.interruptOnCancel together.",
+      "defaultValue": "0ms"
+    },
+    {
+      "key": "spark.sql.thriftserver.scheduler.pool",
+      "doc": "Set a Fair Scheduler pool for a JDBC client session."
+    },
+    {
+      "key": "spark.sql.thriftserver.ui.retainedSessions",
+      "doc": "The number of SQL client sessions kept in the JDBC/ODBC web UI history.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.thriftserver.ui.retainedStatements",
+      "doc": "The number of SQL statements kept in the JDBC/ODBC web UI history.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.timeTravelTimestampKey",
+      "doc": "The option name to specify the time travel timestamp when reading a table.",
+      "defaultValue": "timestampAsOf"
+    },
+    {
+      "key": "spark.sql.timeTravelVersionKey",
+      "doc": "The option name to specify the time travel table version when reading a table.",
+      "defaultValue": "versionAsOf"
+    },
+    {
+      "key": "spark.sql.timestampType",
+      "doc": "Configures the default timestamp type of Spark SQL, including SQL DDL, Cast clause, type literal and the schema inference of data sources. Setting the configuration as TIMESTAMP_NTZ will use TIMESTAMP WITHOUT TIME ZONE as the default type while putting it as TIMESTAMP_LTZ will use TIMESTAMP WITH LOCAL TIME ZONE. Before the 3.4.0 release, Spark only supports the TIMESTAMP WITH LOCAL TIME ZONE type.",
+      "defaultValue": "TIMESTAMP_LTZ"
+    },
+    {
+      "key": "spark.sql.transposeMaxValues",
+      "doc": "When doing a transpose without specifying values for the index column this is the maximum number of values that will be transposed without error.",
+      "defaultValue": "500"
+    },
+    {
+      "key": "spark.sql.truncateTable.ignorePermissionAcl.enabled",
+      "doc": "When set to true, TRUNCATE TABLE command will not try to set back original permission and ACLs when re-creating the table/partition paths.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.tvf.allowMultipleTableArguments.enabled",
+      "doc": "When true, allows multiple table arguments for table-valued functions, receiving the cartesian product of all the rows of these tables.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.udt.allowCreatingUDTFromString",
+      "doc": "When true, Spark loads and instantiates the UserDefinedType class named in a schema string (for example the schema stored in Parquet/ORC file metadata) while inferring or parsing a schema. Because the class name is taken from the data being read, a crafted file can make Spark load an arbitrary class from the classpath. Set this to false to block loading UDT classes by name, optionally allowing specific classes via 'spark.sql.udt.allowedDynamicUDTClasses'.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.udt.allowedDynamicUDTClasses",
+      "doc": "When 'spark.sql.udt.allowCreatingUDTFromString' is false, UserDefinedType classes listed here (by fully qualified class name) may still be loaded and instantiated from a schema string. Has no effect when UDT loading is enabled.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.ui.explainMode",
+      "doc": "Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.",
+      "defaultValue": "formatted"
+    },
+    {
+      "key": "spark.sql.ui.retainedExecutions",
+      "doc": "Number of executions to retain in the Spark UI.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.useCommonExprIdForAlias",
+      "doc": "When true, use the common expression ID for the alias when rewriting With expressions. Otherwise, use the index of the common expression definition. When true this avoids duplicate alias names, but is helpful to set to false for testing to ensure that alias names are consistent.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variable.substitute",
+      "doc": "This enables substitution using syntax like `${var}`, `${system:var}`, and `${env:var}`.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variant.allowDuplicateKeys",
+      "doc": "When set to false, parsing variant from JSON will throw an error if there are duplicate keys in the input JSON object. When set to true, the parser will keep the last occurrence of all fields with the same key.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.variant.allowReadingShredded",
+      "doc": "When true, the Parquet reader is allowed to read shredded or unshredded variant. When false, it only reads unshredded variant.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.variant.forceShreddingSchemaForTest",
+      "doc": "FOR INTERNAL TESTING ONLY. Sets shredding schema for Variant.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.variant.pushVariantIntoScan",
+      "doc": "When true, replace variant type in the scan schema with a struct containing requested fields.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.variant.writeShredding.enabled",
+      "doc": "When true, the Parquet writer is allowed to write shredded variant. ",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.view.maxNestedViewDepth",
+      "doc": "The maximum depth of a view reference in a nested view. A nested view may reference other nested views, the dependencies are organized in a directed acyclic graph (DAG). However the DAG depth may become too large and cause unexpected behavior. This configuration puts a limit on this: when the depth of a view exceeds this value during analysis, we terminate the resolution to avoid potential errors.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.warehouse.dir",
+      "doc": "The default location for managed databases and tables.",
+      "defaultValue": null,
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the window operator",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by window operator",
+      "defaultValue": "2147483647"
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/data/spark_config.json b/crates/sail-spark-connect/data/config/spark-4.1.json
similarity index 94%
rename from crates/sail-spark-connect/data/spark_config.json
rename to crates/sail-spark-connect/data/config/spark-4.1.json
index 292c37249..2121bd631 100644
--- a/crates/sail-spark-connect/data/spark_config.json
+++ b/crates/sail-spark-connect/data/config/spark-4.1.json
@@ -1,7 +1,230 @@
 {
-  "sparkVersion": "4.1.1",
+  "sparkVersion": "4.1.3",
   "comment": "This is a generated file. Do not edit it directly.",
   "entries": [
+    {
+      "key": "spark.connect.authenticate.token",
+      "doc": "A pre-shared token that will be used to authenticate clients. This secret must be passed as a bearer token by for clients to connect.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.copyFromLocalToFs.allowDestLocal",
+      "doc": "\n(Deprecated since Spark 4.0, please set\n'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead.\n",
+      "defaultValue": "false",
+      "isStatic": true,
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Use 'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead."
+      }
+    },
+    {
+      "key": "spark.connect.execute.manager.abandonedTombstonesSize",
+      "doc": "Maximum size of the cache of abandoned executions.",
+      "defaultValue": "10000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.detachedTimeout",
+      "doc": "Timeout after which executions without an attached RPC will be removed.",
+      "defaultValue": "5m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.maintenanceInterval",
+      "doc": "Interval at which execution manager will search for abandoned executions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.enabled",
+      "doc": "Enables reattachable execution on the server. If disabled and a client requests it, non-reattachable execution will follow and should run until query completion. This will work, unless there is a GRPC stream error, in which case the client will discover that execution is not reattachable when trying to reattach fails.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.observerRetryBufferSize",
+      "doc": "For reattachable execution, the total size of responses that were already sent to be kept in the buffer in case of connection error and client needing to retry. Set 0 to don't buffer anything (even last sent response).With any value greater than 0, the last sent response will always be buffered.",
+      "defaultValue": "10m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamDuration",
+      "doc": "For reattachable execution, after this amount of time the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "2m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamSize",
+      "doc": "For reattachable execution, after total responses size exceeds this value, the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "1g",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.command.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.CommandPlugin to support custom\nCommand types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.expression.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.ExpressionPlugin to support custom\nExpression types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.relation.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.RelationPlugin to support custom\nRelation types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.arrow.maxBatchSize",
+      "doc": "When using Apache Arrow, limit the maximum size of one arrow batch, in bytes unless otherwise specified, that can be sent from server side to client side. Currently, we conservatively use 70% of it because the size is not accurate but estimated.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.address",
+      "doc": "The address for Spark Connect server to bind.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.port",
+      "doc": "The port for Spark Connect server to bind.",
+      "defaultValue": "15002",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.interceptor.classes",
+      "doc": "Comma separated list of class names that must implement the io.grpc.ServerInterceptor interface.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.marshallerRecursionLimit",
+      "doc": "\nSets the recursion limit to grpc protobuf messages.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxInboundMessageSize",
+      "doc": "Sets the maximum inbound message in bytes size for the gRPC requests.Requests with a larger payload will fail.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxMetadataSize",
+      "doc": "Sets the maximum size of metadata fields. For instance, it restricts metadata fields in `ErrorInfo`.",
+      "defaultValue": "1024b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.port.maxRetries",
+      "doc": "The max port retry attempts for the gRPC server binding.By default, it's set to 0, and the server will fail fast in case of port conflicts.",
+      "defaultValue": "0",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.jvmStacktrace.maxSize",
+      "doc": "\nSets the maximum stack trace size to display when\n`spark.sql.pyspark.jvmStacktrace.enabled` is true.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.maxPlanSize",
+      "doc": "The maximum size of a (decompressed) proto plan that can be executed in Spark Connect. If the size of the plan exceeds this limit, an error will be thrown. The size is in bytes.",
+      "defaultValue": "536870912b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.ml.backend.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.MLBackendPlugin to replace the\nspecified Spark ML operators with a backend-specific implementation.\n",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.connect.progress.reportInterval",
+      "doc": "The interval at which the progress of a query is reported to the client. If the value is set to a negative value the progress reports will be disabled.",
+      "defaultValue": "2s"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.enabled",
+      "doc": "Enables ML cache memory control, it includes offloading model to disk, limiting model size, and limiting per-session ML cache size.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.maxInMemorySize",
+      "doc": "Maximum in-memory size of the MLCache per session. The cache will offload the least recently used models to Spark driver local disk if the size exceeds this limit. The size is in bytes.",
+      "defaultValue": "268435456b"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.maxModelSize",
+      "doc": "Maximum size of a single SparkML model. The size is in bytes.",
+      "defaultValue": "1g"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.maxStorageSize",
+      "doc": "Maximum total size (including in-memory and offloaded data) of the ml cache. The size is in bytes.",
+      "defaultValue": "10g"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.offloadingTimeout",
+      "doc": "Timeout of model offloading in MLCache. Models will be offloaded to Spark driver local disk if they are not used for this amount of time. The timeout is in minutes.",
+      "defaultValue": "900000ms"
+    },
+    {
+      "key": "spark.connect.session.inactiveOperations.cacheExpiration",
+      "doc": "Expiration time for inactive operation IDs cache in Spark Connect Session. Operations are cached after completion for a period of time to detect duplicates. The time should allow for network late arrivals, at least several minutes.",
+      "defaultValue": "1800000ms",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.closedSessionsTombstonesSize",
+      "doc": "Maximum size of the cache of sessions after which sessions that did not receive any requests will be removed.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.defaultSessionTimeout",
+      "doc": "Timeout after which sessions without any new incoming RPC will be removed. Setting it to -1 indicates that sessions should be kept forever.",
+      "defaultValue": "60m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.maintenanceInterval",
+      "doc": "Interval at which session manager will search for expired sessions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.planCache.alwaysCacheDataSourceReadsEnabled",
+      "doc": "When true, always cache the translation of Read.DataSource plans in the plan cache. This massively improves the performance of queries that reuse the same Read.DataSource within the same session, since these translations/analyses are usually quite costly.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.planCache.enabled",
+      "doc": "When true, the cache of resolved logical plans is enabled if 'spark.connect.session.planCache.maxSize' is greater than zero. When false, the cache is disabled even if 'spark.connect.session.planCache.maxSize' is greater than zero. The caching is best-effort and not guaranteed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.planCache.maxSize",
+      "doc": "Sets the maximum number of cached resolved logical plans in Spark Connect Session. If set to a value less or equal than zero will disable the plan cache.",
+      "defaultValue": "32",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.planCompression.defaultAlgorithm",
+      "doc": "The default algorithm of proto plan compression.",
+      "defaultValue": "ZSTD"
+    },
+    {
+      "key": "spark.connect.session.planCompression.threshold",
+      "doc": "The threshold in bytes for the size of proto plan to be compressed. If the size of proto plan is smaller than this threshold, it will not be compressed. Set to -1 to disable plan compression.",
+      "defaultValue": "10485760"
+    },
+    {
+      "key": "spark.connect.session.resultChunking.maxChunkSize",
+      "doc": "The max size of a chunk in responses for a result batch. Result chunking is enabled if this config is set to a value greater than 0 and if the client allows it in ResultChunkingOptions. Otherwise, for example if set to -1, this feature is disabled. While spark.connect.grpc.arrow.maxBatchSize determines the max size of a result batch, maxChunkSize defines the max size of each individual chunk that is part of the batch that will be sent in a response. This allows the server to send large rows to clients. The size is in bytes.",
+      "defaultValue": "120795955b"
+    },
     {
       "key": "spark.python.sql.dataFrameDebugging.enabled",
       "doc": "Enable the DataFrame debugging. This feature is enabled by default, but has a non-trivial performance overhead because of the stack trace collection.",
@@ -609,11 +832,33 @@
       "doc": "When true, use OffHeapColumnVector in ColumnarBatch.",
       "fallback": "spark.memory.offHeap.enabled"
     },
+    {
+      "key": "spark.sql.connect.enrichError.enabled",
+      "doc": "\nWhen true, it enriches errors with full exception messages and optionally server-side\nstacktrace on the client side via an additional RPC.\n",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.connect.serverStacktrace.enabled",
+      "doc": "When true, it sets the server-side stacktrace in the user-facing Spark exception.",
+      "defaultValue": "true"
+    },
     {
       "key": "spark.sql.connect.shuffleDependency.fileCleanup.enabled",
       "doc": "When enabled, shuffle files will be cleaned up at the end of Spark Connect SQL executions.",
       "fallback": "spark.sql.shuffleDependency.fileCleanup.enabled"
     },
+    {
+      "key": "spark.sql.connect.ui.retainedSessions",
+      "doc": "The number of client sessions kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedStatements",
+      "doc": "The number of statements kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
     {
       "key": "spark.sql.constraintPropagation.enabled",
       "doc": "When true, the query optimizer will infer and propagate data constraints in the query plan to optimize them. Constraint propagation can sometimes be computationally expensive for certain kinds of query plans (such as those with a large number of predicates and aliases) which might negatively impact overall runtime.",
@@ -2460,6 +2705,11 @@
       "doc": "For IN predicate, Parquet filter will push-down a set of OR clauses if its number of values not exceeds this threshold. Otherwise, Parquet filter will push-down a value greater than or equal to its minimum value and less than or equal to its maximum value. By setting this value to 0 this feature can be disabled. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
       "defaultValue": "10"
     },
+    {
+      "key": "spark.sql.parquet.reader.respectUnknownTypeAnnotation.enabled",
+      "doc": "When enabled, respects the UNKNOWN type annotation in Parquet files during schema inference and infers NullType. When disabled, ignores the UNKNOWN annotation and uses the physical type instead.",
+      "defaultValue": "false"
+    },
     {
       "key": "spark.sql.parquet.recordLevelFilter.enabled",
       "doc": "If true, enables Parquet's native record-level filtering using the pushed down filters. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and the vectorized reader is not used. You can ensure the vectorized reader is not used by setting 'spark.sql.parquet.enableVectorizedReader' to false.",
@@ -3508,6 +3758,16 @@
       "doc": "When true, allows multiple table arguments for table-valued functions, receiving the cartesian product of all the rows of these tables.",
       "defaultValue": "false"
     },
+    {
+      "key": "spark.sql.udt.allowCreatingUDTFromString",
+      "doc": "When true, Spark loads and instantiates the UserDefinedType class named in a schema string (for example the schema stored in Parquet/ORC file metadata) while inferring or parsing a schema. Because the class name is taken from the data being read, a crafted file can make Spark load an arbitrary class from the classpath. Set this to false to block loading UDT classes by name, optionally allowing specific classes via 'spark.sql.udt.allowedDynamicUDTClasses'.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.udt.allowedDynamicUDTClasses",
+      "doc": "When 'spark.sql.udt.allowCreatingUDTFromString' is false, UserDefinedType classes listed here (by fully qualified class name) may still be loaded and instantiated from a schema string. Has no effect when UDT loading is enabled.",
+      "defaultValue": ""
+    },
     {
       "key": "spark.sql.ui.explainMode",
       "doc": "Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.",
diff --git a/crates/sail-spark-connect/data/config/spark-4.2.json b/crates/sail-spark-connect/data/config/spark-4.2.json
new file mode 100644
index 000000000..3273eb7c6
--- /dev/null
+++ b/crates/sail-spark-connect/data/config/spark-4.2.json
@@ -0,0 +1,4205 @@
+{
+  "sparkVersion": "4.2.0",
+  "comment": "This is a generated file. Do not edit it directly.",
+  "entries": [
+    {
+      "key": "spark.connect.authenticate.token",
+      "doc": "A pre-shared token that will be used to authenticate clients. This secret must be passed as a bearer token by for clients to connect.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.copyFromLocalToFs.allowDestLocal",
+      "doc": "\n(Deprecated since Spark 4.0, please set\n'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead.\n",
+      "defaultValue": "false",
+      "isStatic": true,
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Use 'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead."
+      }
+    },
+    {
+      "key": "spark.connect.execute.manager.abandonedTombstonesSize",
+      "doc": "Maximum size of the cache of abandoned executions.",
+      "defaultValue": "10000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.detachedTimeout",
+      "doc": "Timeout after which executions without an attached RPC will be removed.",
+      "defaultValue": "5m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.manager.maintenanceInterval",
+      "doc": "Interval at which execution manager will search for abandoned executions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.enabled",
+      "doc": "Enables reattachable execution on the server. If disabled and a client requests it, non-reattachable execution will follow and should run until query completion. This will work, unless there is a GRPC stream error, in which case the client will discover that execution is not reattachable when trying to reattach fails.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.observerRetryBufferSize",
+      "doc": "For reattachable execution, the total size of responses that were already sent to be kept in the buffer in case of connection error and client needing to retry. Set 0 to don't buffer anything (even last sent response).With any value greater than 0, the last sent response will always be buffered.",
+      "defaultValue": "10m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamDuration",
+      "doc": "For reattachable execution, after this amount of time the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "2m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.execute.reattachable.senderMaxStreamSize",
+      "doc": "For reattachable execution, after total responses size exceeds this value, the response stream will be automatically completed and client needs to send a new ReattachExecute RPC to continue. Set to 0 for unlimited.",
+      "defaultValue": "1g",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.command.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.CommandPlugin to support custom\nCommand types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.expression.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.ExpressionPlugin to support custom\nExpression types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.getStatus.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.GetStatusPlugin to support custom\nGetStatus extensions in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.extensions.relation.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.RelationPlugin to support custom\nRelation types in proto.\n",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.arrow.maxBatchSize",
+      "doc": "When using Apache Arrow, limit the maximum size of one arrow batch, in bytes unless otherwise specified, that can be sent from server side to client side. Currently, we conservatively use 70% of it because the size is not accurate but estimated.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.address",
+      "doc": "The address for Spark Connect server to bind.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.binding.port",
+      "doc": "The port for Spark Connect server to bind.",
+      "defaultValue": "15002",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.interceptor.classes",
+      "doc": "Comma separated list of class names that must implement the io.grpc.ServerInterceptor interface.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.marshallerRecursionLimit",
+      "doc": "\nSets the recursion limit to grpc protobuf messages.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxInboundMessageSize",
+      "doc": "Sets the maximum inbound message in bytes size for the gRPC requests.Requests with a larger payload will fail.",
+      "defaultValue": "134217728b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.maxMetadataSize",
+      "doc": "Sets the maximum size of metadata fields. For instance, it restricts metadata fields in `ErrorInfo`.",
+      "defaultValue": "1024b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.grpc.port.maxRetries",
+      "doc": "The max port retry attempts for the gRPC server binding.By default, it's set to 0, and the server will fail fast in case of port conflicts.",
+      "defaultValue": "0",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.jvmStacktrace.maxSize",
+      "doc": "\nSets the maximum stack trace size to display when\n`spark.sql.pyspark.jvmStacktrace.enabled` is true.\n",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.maxPlanSize",
+      "doc": "The maximum size of a (decompressed) proto plan that can be executed in Spark Connect. If the size of the plan exceeds this limit, an error will be thrown. The size is in bytes.",
+      "defaultValue": "536870912b",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.ml.backend.classes",
+      "doc": "\nComma separated list of classes that implement the trait\norg.apache.spark.sql.connect.plugin.MLBackendPlugin to replace the\nspecified Spark ML operators with a backend-specific implementation.\n",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.connect.progress.reportInterval",
+      "doc": "The interval at which the progress of a query is reported to the client. If the value is set to a negative value the progress reports will be disabled.",
+      "defaultValue": "2s"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.enabled",
+      "doc": "Enables ML cache memory control, it includes offloading model to disk, limiting model size, and limiting per-session ML cache size.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.maxInMemorySize",
+      "doc": "Maximum in-memory size of the MLCache per session. The cache will offload the least recently used models to Spark driver local disk if the size exceeds this limit. The size is in bytes.",
+      "defaultValue": "268435456b"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.maxModelSize",
+      "doc": "Maximum size of a single SparkML model. The size is in bytes.",
+      "defaultValue": "1g"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.maxStorageSize",
+      "doc": "Maximum total size (including in-memory and offloaded data) of the ml cache. The size is in bytes.",
+      "defaultValue": "10g"
+    },
+    {
+      "key": "spark.connect.session.connectML.mlCache.memoryControl.offloadingTimeout",
+      "doc": "Timeout of model offloading in MLCache. Models will be offloaded to Spark driver local disk if they are not used for this amount of time. The timeout is in minutes.",
+      "defaultValue": "900000ms"
+    },
+    {
+      "key": "spark.connect.session.inactiveOperations.cacheExpiration",
+      "doc": "Expiration time for inactive operation IDs cache in Spark Connect Session. Operations are cached after completion for a period of time to detect duplicates. The time should allow for network late arrivals, at least several minutes.",
+      "defaultValue": "1800000ms",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.closedSessionsTombstonesSize",
+      "doc": "Maximum size of the cache of sessions after which sessions that did not receive any requests will be removed.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.defaultSessionTimeout",
+      "doc": "Timeout after which sessions without any new incoming RPC will be removed. Setting it to -1 indicates that sessions should be kept forever.",
+      "defaultValue": "60m",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.manager.maintenanceInterval",
+      "doc": "Interval at which session manager will search for expired sessions to remove.",
+      "defaultValue": "30s",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.planCache.alwaysCacheDataSourceReadsEnabled",
+      "doc": "When true, always cache the translation of Read.DataSource plans in the plan cache. This massively improves the performance of queries that reuse the same Read.DataSource within the same session, since these translations/analyses are usually quite costly.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.planCache.enabled",
+      "doc": "When true, the cache of resolved logical plans is enabled if 'spark.connect.session.planCache.maxSize' is greater than zero. When false, the cache is disabled even if 'spark.connect.session.planCache.maxSize' is greater than zero. The caching is best-effort and not guaranteed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.connect.session.planCache.maxSize",
+      "doc": "Sets the maximum number of cached resolved logical plans in Spark Connect Session. If set to a value less or equal than zero will disable the plan cache.",
+      "defaultValue": "32",
+      "isStatic": true
+    },
+    {
+      "key": "spark.connect.session.planCompression.defaultAlgorithm",
+      "doc": "The default algorithm of proto plan compression.",
+      "defaultValue": "ZSTD"
+    },
+    {
+      "key": "spark.connect.session.planCompression.threshold",
+      "doc": "The threshold in bytes for the size of proto plan to be compressed. If the size of proto plan is smaller than this threshold, it will not be compressed. Set to -1 to disable plan compression.",
+      "defaultValue": "10485760"
+    },
+    {
+      "key": "spark.connect.session.resultChunking.maxChunkSize",
+      "doc": "The max size of a chunk in responses for a result batch. Result chunking is enabled if this config is set to a value greater than 0 and if the client allows it in ResultChunkingOptions. Otherwise, for example if set to -1, this feature is disabled. While spark.connect.grpc.arrow.maxBatchSize determines the max size of a result batch, maxChunkSize defines the max size of each individual chunk that is part of the batch that will be sent in a response. This allows the server to send large rows to clients. The size is in bytes.",
+      "defaultValue": "120795955b"
+    },
+    {
+      "key": "spark.python.sql.dataFrameDebugging.enabled",
+      "doc": "Enable the DataFrame debugging. This feature is enabled by default, but has a non-trivial performance overhead because of the stack trace collection.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.adaptive.advisoryPartitionSizeInBytes",
+      "doc": "The advisory size in bytes of the shuffle partition during adaptive optimization (when spark.sql.adaptive.enabled is true). It takes effect when Spark coalesces small shuffle partitions or splits skewed shuffle partition.",
+      "fallback": "spark.sql.adaptive.shuffle.targetPostShuffleInputSize"
+    },
+    {
+      "key": "spark.sql.adaptive.applyFinalStageShuffleOptimizations",
+      "doc": "Configures whether adaptive query execution (if enabled) should apply shuffle coalescing and local shuffle read optimization for the final query stage.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.autoBroadcastJoinThreshold",
+      "doc": "Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join. By setting this value to -1 broadcasting can be disabled. The default value is same with spark.sql.autoBroadcastJoinThreshold. Note that, this config is used only in adaptive framework."
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark will coalesce contiguous shuffle partitions according to the target size (specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes'), to avoid too many small tasks.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.initialPartitionNum",
+      "doc": "The initial number of shuffle partitions before coalescing. If not set, it equals to spark.sql.shuffle.partitions. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true."
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.minPartitionNum",
+      "doc": "(deprecated) The suggested (not guaranteed) minimum number of shuffle partitions after coalescing. If not set, the default value is the default parallelism of the Spark cluster. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true.",
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use 'spark.sql.adaptive.coalescePartitions.minPartitionSize' instead."
+      }
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.minPartitionSize",
+      "doc": "The minimum size of shuffle partitions after coalescing. This is useful when the adaptively calculated target size is too small during partition coalescing.",
+      "defaultValue": "1MB"
+    },
+    {
+      "key": "spark.sql.adaptive.coalescePartitions.parallelismFirst",
+      "doc": "When true, Spark does not respect the target size specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes' (default 64MB) when coalescing contiguous shuffle partitions, but adaptively calculate the target size according to the default parallelism of the Spark cluster. The calculated size is usually smaller than the configured target size. This is to maximize the parallelism and avoid performance regressions when enabling adaptive query execution. It's recommended to set this config to false on a busy cluster to make resource utilization more efficient (not many small tasks).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.customCostEvaluatorClass",
+      "doc": "The custom cost evaluator class to be used for adaptive execution. If not being set, Spark will use its own SimpleCostEvaluator by default."
+    },
+    {
+      "key": "spark.sql.adaptive.enabled",
+      "doc": "When true, enable adaptive query execution, which re-optimizes the query plan in the middle of query execution, based on accurate runtime statistics.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.fetchShuffleBlocksInBatch",
+      "doc": "Whether to fetch the contiguous shuffle blocks in batch. Instead of fetching blocks one by one, fetching contiguous shuffle blocks for the same map task in batch can reduce IO and improve performance. Note, multiple contiguous blocks exist in single fetch request only happen when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.coalescePartitions.enabled' are both true. This feature also depends on a relocatable serializer, the concatenation support codec in use, the new version shuffle fetch protocol and io encryption is disabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.forceApply",
+      "doc": "Adaptive query execution is skipped when the query does not have exchanges or sub-queries. By setting this config to true (together with 'spark.sql.adaptive.enabled' set to true), Spark will force apply adaptive query execution for all supported queries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.adaptive.forceOptimizeSkewedJoin",
+      "doc": "When true, force enable OptimizeSkewedJoin even if it introduces extra shuffle.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.adaptive.localShuffleReader.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark tries to use local shuffle reader to read the shuffle data when the shuffle partitioning is not needed, for example, after converting sort-merge join to broadcast-hash join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.logLevel",
+      "doc": "Configures the log level for adaptive execution logging of plan changes. The value can be ERROR, WARN, INFO, DEBUG, TRACE.",
+      "defaultValue": "DEBUG"
+    },
+    {
+      "key": "spark.sql.adaptive.maxShuffledHashJoinLocalMapThreshold",
+      "doc": "Configures the maximum size in bytes per partition that can be allowed to build local hash map. If this value is not smaller than spark.sql.adaptive.advisoryPartitionSizeInBytes and all the partition size are not larger than this config, join selection prefer to use shuffled hash join instead of sort merge join regardless of the value of spark.sql.join.preferSortMergeJoin.",
+      "defaultValue": "0b"
+    },
+    {
+      "key": "spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin",
+      "doc": "The relation with a non-empty partition ratio lower than this config will not be considered as the build side of a broadcast-hash join in adaptive execution regardless of its size.This configuration only has an effect when 'spark.sql.adaptive.enabled' is true.",
+      "defaultValue": "0.2"
+    },
+    {
+      "key": "spark.sql.adaptive.optimizeSkewsInRebalancePartitions.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark will optimize the skewed shuffle partitions in RebalancePartitions and split them to smaller ones according to the target size (specified by 'spark.sql.adaptive.advisoryPartitionSizeInBytes'), to avoid data skew.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.optimizer.excludedRules",
+      "doc": "Configures a list of rules to be disabled in the adaptive optimizer, in which the rules are specified by their rule names and separated by comma. The optimizer will log the rules that have indeed been excluded."
+    },
+    {
+      "key": "spark.sql.adaptive.rebalancePartitionsSmallPartitionFactor",
+      "doc": "A partition will be merged during splitting if its size is small than this factor multiply spark.sql.adaptive.advisoryPartitionSizeInBytes.",
+      "defaultValue": "0.2"
+    },
+    {
+      "key": "spark.sql.adaptive.shuffle.targetPostShuffleInputSize",
+      "doc": "(Deprecated since Spark 3.0)",
+      "defaultValue": "64MB",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.adaptive.advisoryPartitionSizeInBytes' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.enabled",
+      "doc": "When true and 'spark.sql.adaptive.enabled' is true, Spark dynamically handles skew in shuffled join (sort-merge and shuffled hash) by splitting (and replicating if needed) skewed partitions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.skewedPartitionFactor",
+      "doc": "A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than 'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'",
+      "defaultValue": "5.0"
+    },
+    {
+      "key": "spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes",
+      "doc": "A partition is considered as skewed if its size in bytes is larger than this threshold and also larger than 'spark.sql.adaptive.skewJoin.skewedPartitionFactor' multiplying the median partition size. Ideally this config should be set larger than 'spark.sql.adaptive.advisoryPartitionSizeInBytes'.",
+      "defaultValue": "256MB"
+    },
+    {
+      "key": "spark.sql.adaptive.streaming.stateless.enabled",
+      "doc": "When true, enable adaptive query execution for stateless streaming query. To enable this config, `spark.sql.adaptive.enabled` needs to be also enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.addPartitionInBatch.size",
+      "doc": "The number of partitions to be handled in one turn when use `AlterTableAddPartitionCommand` or `RepairTableCommand` to add partitions into table. The smaller batch size is, the less memory is required for the real handler, e.g. Hive Metastore.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.allowNamedFunctionArguments",
+      "doc": "If true, Spark will turn on support for named parameters for all functions that has it implemented.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.alwaysInlineCommonExpr",
+      "doc": "When true, always inline common expressions instead of using the WITH expression. This may lead to duplicated expressions and the config should only be enabled if you hit bugs caused by the WITH expression.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.allowSubqueryExpressionsInLambdasOrHigherOrderFunctions",
+      "doc": "When set to false, the analyzer will throw an error if a subquery expression appears in a lambda function or higher-order function. When set to true, it restores the legacy behavior of allowing subquery eexpressions in lambda functions or higher-order functions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.canonicalization.multiCommutativeOpMemoryOptThreshold",
+      "doc": "The minimum number of operands in a commutative expression tree to invoke the MultiCommutativeOp memory optimization during canonicalization.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.analyzer.dontDeduplicateExpressionIfExprIdInOutput",
+      "doc": "DeduplicateRelations shouldn't remap expressions to new ExprIds if old ExprId still exists in output.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.expandTagPassthroughDuplicates",
+      "doc": "When true, Expand tags pass-through child attributes that share a name with a grouping attribute using __is_duplicate metadata, so that name-based resolution against the Expand output does not produce AMBIGUOUS_REFERENCE errors.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.failAmbiguousSelfJoin",
+      "doc": "When true, fail the Dataset query if it contains ambiguous self-join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.lowerEmptyGroupingSetToGlobalAggregate.enabled",
+      "doc": "When true, a grand-total GROUP BY GROUPING SETS (()) (and the equivalent empty CUBE() / ROLLUP()) is lowered to a global aggregate during analysis, so it returns one row over empty input, matching an aggregation with no GROUP BY clause. When false, falls back to the legacy Expand-based lowering that returns no rows over empty input.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.maxIterations",
+      "doc": "The max number of iterations the analyzer runs.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.analyzer.preferColumnOverLcaInArrayIndex",
+      "doc": "When true, prefer the column from the underlying relation over the lateral column alias reference with the same name (see SPARK-53734).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.scalarSubqueryAllowGroupByColumnEqualToConstant",
+      "doc": "When set to true, allow scalar subqueries with group-by on a column that also  has an equality filter with a constant (SPARK-48557).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.dualRunSampleRate",
+      "doc": "Represents the rate of queries that will be run in both fixed-point and single-pass mode (dual run). It should be taken into account that the sample rate is not a strict percentage (in tests we don't sample). It is determined whether query should be run in dual run mode by comparing a random value with the value of this flag.",
+      "defaultValue": "0.001"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.dualRunWithLegacy",
+      "doc": "When true, run both analyzers to check if single-pass Analyzer correctly produces the same analyzed plan as the fixed-point Analyzer for the existing set of features defined in the ResolverGuard",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.enabled",
+      "doc": "When true, use the single-pass Resolver instead of the fixed-point Analyzer. This is an alternative Analyzer framework, which resolves the parsed logical plan in a single post-order traversal. It uses ExpressionResolver to resolve expressions and NameScope to control the visibility of names. In contrast to the current fixed-point framework, subsequent in-tree traversals are disallowed. Most of the fixed-point Analyzer code is reused in the form of specific node transformation functions (AliasResolution.resolve, FunctionResolution.resolveFunction, etc). This feature is currently under development.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.enabledTentatively",
+      "doc": "When true, use the single-pass Resolver instead of the fixed-point Analyzer only if a SQL query or a DataFrame program is fully supported by the single-pass Analyzer. This is an alternative Analyzer framework, which resolves the parsed logical plan in a single post-order traversal. It uses ExpressionResolver to resolve expressions and NameScope to control the visibility of names. In contrast to the current fixed-point framework, subsequent in-tree traversals are disallowed. Most of the fixed-point Analyzer code is reused in the form of specific node transformation functions (AliasResolution.resolve, FunctionResolution.resolveFunction, etc).This feature is currently under development.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.exposeResolverGuardFailure",
+      "doc": "When true, any failure thrown from ResolverGuard will be exposed as a query failure. Otherwise we just assume that the ResolverGuard returned false and the query is not supported by the single-pass Analyzer. This is important to make dual-runs unnoticeable in production.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.preventUsingAliasesFromNonDirectChildren",
+      "doc": "When true, in Sort/Having/Filter expressions allow replacing of these expressions only with semantically equal aliased expressions from direct children. This is necessary in order to stay compatible with fixed-point, but the functionality and correctness remain the same. Because enabling this case would break some cases that are supported in single-pass but not in fixed-point, this flag should only be used to hide false positive logical plan mismatches during testing.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.relationBridging.enabled",
+      "doc": "When set to true, the single-pass Resolver will reuse the relation metadata that was previously resolved in fixed-point run. This makes sense only in ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER mode. In that case HybridAnalyzer enables the AnalyzerBridgeState and passes it to the single-pass Analyzer after the fixed-point run is complete. Single-pass Resolver uses this AnalyzerBridgeState to construct a special RelationMetadataProvider implementation - BridgedRelationMetadataProvider. This component simply reuses cached relation metadata and avoids any blocking calls (catalog RPCs or table metadata reads).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.returnSinglePassResultInDualRun",
+      "doc": "When true, return the result of the single-pass resolver as the result of the dual run analysis (which is used if the ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER flag value is true). Otherwise, return the result of the fixed-point analyzer.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.runExtendedResolutionChecks",
+      "doc": "When true, run `extendedResolutionChecks` after the main analysis.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.runHeavyExtendedResolutionChecks",
+      "doc": "When true, run heavy `extendedResolutionChecks` after the main analysis. Otherwise skip them. Heavy check either involves a network call changing external persistent storage, or changes a global state. For example, `ViewSyncSchemaToMetaStore` calls alter table.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.throwFromResolverGuard",
+      "doc": "When set to true, ResolverGuard will throw a descriptive error on unsupported features.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.analyzer.singlePassResolver.validationEnabled",
+      "doc": "When true, validate the Resolver output with ResolutionValidator. The ResolutionValidator validates the resolved logical plan tree in one pass and asserts the internal contracts. It uses the ExpressionResolutionValidator internally to validate resolved expression trees in the same manner.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.sqlFunctionResolution.applyConfOverrides",
+      "doc": "When true, applies the conf overrides for certain feature flags during the resolution of user-defined sql table valued functions, consistent with view resolution.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.strictDataFrameColumnResolution",
+      "doc": "When true (default), enforce strict resolution of Spark Connect DataFrame columns (UnresolvedAttribute carrying a plan id tag) via plan-id-based resolution. When false, also try name-based resolution as a fallback for tagged attributes.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.subqueryAliasAlwaysPropagateMetadataColumns",
+      "doc": "When true, SubqueryAlias always propagates metadata columns from its child. When false, SubqueryAlias only propagates metadata columns if the child is a LeafNode or another SubqueryAlias (legacy behavior).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.unionIsResolvedWhenDuplicatesPerChildResolved",
+      "doc": "When true, union should only be resolved once there are no duplicate attributes in each branch.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.analyzer.uniqueNecessaryMetadataColumns",
+      "doc": "When this conf is enabled, AddMetadataColumns rule should only add necessary metadata columns and only if those columns are not already present in the project list.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.ansi.doubleQuotedIdentifiers",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, Spark SQL reads literals enclosed in double quoted (\") as identifiers. When false they are read as string literals.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.enabled",
+      "doc": "When true, Spark SQL uses an ANSI compliant dialect instead of being Hive compliant. For example, Spark will throw an exception at runtime instead of returning null results when the inputs to a SQL operator/function are invalid. For full details of this dialect, you can find them in the section \"ANSI Compliance\" of Spark's documentation. Some ANSI dialect features may be not from the ANSI SQL standard directly, but their behaviors align with ANSI SQL's style",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.ansi.enforceReservedKeywords",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, the Spark SQL parser enforces the ANSI reserved keywords and forbids SQL queries that use reserved keywords as alias names and/or identifiers for table, view, function, etc.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.relationPrecedence",
+      "doc": "When true and 'spark.sql.ansi.enabled' is true, JOIN takes precedence over comma when combining relation. For example, `t1, t2 JOIN t3` should result to `t1 X (t2 X t3)`. If the config is false, the result is `(t1 X t2) X t3`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.ansi.strictIndexOperator",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.4.0",
+        "comment": "This was an internal configuration. It is not needed anymore since Spark SQL always returns null when getting a map value with a non-existing key. See SPARK-40066 for more details."
+      }
+    },
+    {
+      "key": "spark.sql.artifact.cacheStorageLevel",
+      "doc": "Storage level for cached blocks in artifact manager. Valid values are any StorageLevel name (e.g., MEMORY_AND_DISK_SER, DISK_ONLY, MEMORY_ONLY, etc.).",
+      "defaultValue": "MEMORY_AND_DISK_SER"
+    },
+    {
+      "key": "spark.sql.artifact.copyFromLocalToFs.allowDestLocal",
+      "doc": "\nAllow `spark.copyFromLocalToFs` destination to be local file system\n path on spark driver node when\n`spark.sql.artifact.copyFromLocalToFs.allowDestLocal` is true.\nThis will allow user to overwrite arbitrary file on spark\ndriver node we should only enable it for testing purpose.\n"
+    },
+    {
+      "key": "spark.sql.artifact.isolation.alwaysApplyClassloader",
+      "doc": "When enabled, the classloader holding per-session artifacts will always be applied during SQL executions (useful for Spark Connect). When disabled, the classloader will be applied only when any artifact is added to the session.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.artifact.isolation.enabled",
+      "doc": "When enabled for a Spark Session, artifacts (such as JARs, files, archives) added to this session are isolated from other sessions within the same Spark instance. When disabled for a session, artifacts added to this session are visible to other sessions that have this config disabled. This config can only be set during the creation of a Spark Session and will have no effect when changed in the middle of session usage.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.assumeAnsiFalseIfNotPersisted.enabled",
+      "doc": "If enabled, assume ANSI mode is false if not persisted during view or UDF creation. Otherwise use the default value.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.autoBroadcastJoinThreshold",
+      "doc": "Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when performing a join. By setting this value to -1 broadcasting can be disabled.",
+      "defaultValue": "10MB"
+    },
+    {
+      "key": "spark.sql.avro.compression.codec",
+      "doc": "Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate, snappy, bzip2, xz and zstandard. Default codec is snappy.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.avro.datetimeRebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files. When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. When EXCEPTION, Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Avro files is unknown.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.avro.datetimeRebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files. When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. When EXCEPTION, Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.avro.deflate.level",
+      "doc": "Compression level for the deflate codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level in the current implementation.",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.avro.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to Avro datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.avro.xz.level",
+      "doc": "Compression level for the xz codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive The default value is 6.",
+      "defaultValue": "6"
+    },
+    {
+      "key": "spark.sql.avro.zstandard.bufferPool.enabled",
+      "doc": "If true, enable buffer pool of ZSTD JNI library when writing of AVRO files",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.avro.zstandard.level",
+      "doc": "Compression level for the zstandard codec used in writing of AVRO files. ",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.binaryOutputStyle",
+      "doc": "The output style used display binary data. Valid values are 'UTF-8', 'BASIC', 'BASE64', 'HEX', and 'HEX_DISCRETE'."
+    },
+    {
+      "key": "spark.sql.broadcastExchange.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to fetch and broadcast the table. If we encounter memory issue like frequently full GC or OOM when broadcast table we can decrease this number in order to reduce memory usage. Notice the number should be carefully chosen since decreasing parallelism might cause longer waiting for other broadcasting. Also, increasing parallelism may cause memory problem.",
+      "defaultValue": "128",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.broadcastTimeout",
+      "doc": "Timeout in seconds for the broadcast wait time in broadcast joins.",
+      "defaultValue": "300"
+    },
+    {
+      "key": "spark.sql.bucketing.coalesceBucketsInJoin.enabled",
+      "doc": "When true, if two bucketed tables with the different number of buckets are joined, the side with a bigger number of buckets will be coalesced to have the same number of buckets as the other side. Bigger number of buckets is divisible by the smaller number of buckets. Bucket coalescing is applied to sort-merge joins and shuffled hash join. Note: Coalescing bucketed table can avoid unnecessary shuffling in join, but it also reduces parallelism and could possibly cause OOM for shuffled hash join.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.bucketing.coalesceBucketsInJoin.maxBucketRatio",
+      "doc": "The ratio of the number of two buckets being coalesced should be less than or equal to this value for bucket coalescing to be applied. This configuration only has an effect when 'spark.sql.bucketing.coalesceBucketsInJoin.enabled' is set to true.",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.cache.serializer",
+      "doc": "The name of a class that implements org.apache.spark.sql.columnar.CachedBatchSerializer. It will be used to translate SQL data into a format that can more efficiently be cached. The underlying API is subject to change so use with caution. Multiple classes cannot be specified. The class must have a no-arg constructor.",
+      "defaultValue": "org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the cartesian product operator",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.spill.size.threshold",
+      "doc": "Threshold for size of rows to be spilled by cartesian product operator",
+      "fallback": "spark.shuffle.spill.maxSizeInBytesForSpillThreshold"
+    },
+    {
+      "key": "spark.sql.cartesianProductExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by cartesian product operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.caseSensitive",
+      "doc": "Whether the query analyzer should be case sensitive or not. Default to case insensitive. It is highly discouraged to turn on case sensitive mode.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.catalog.spark_catalog",
+      "doc": "A catalog implementation that will be used as the v2 interface to Spark's built-in v1 catalog: spark_catalog. This catalog shares its identifier namespace with the spark_catalog and must be consistent with it; for example, if a table can be loaded by the spark_catalog, this catalog must also return the table metadata. To delegate operations to the spark_catalog, implementations can extend 'CatalogExtension'. The value should be either 'builtin' which represents the spark's builit-in V2SessionCatalog, or a fully qualified class name of the catalog implementation.",
+      "defaultValue": "builtin"
+    },
+    {
+      "key": "spark.sql.catalog.spark_catalog.defaultDatabase",
+      "doc": "The default database for session catalog.",
+      "defaultValue": "default",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.catalogImplementation",
+      "doc": "",
+      "defaultValue": "in-memory",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.cbo.enabled",
+      "doc": "Enables CBO for estimation of plan statistics when set true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.card.weight",
+      "doc": "The weight of the ratio of cardinalities (number of rows) in the cost comparison function. The ratio of sizes in bytes has weight 1 - this value. The weighted geometric mean of these ratios is used to decide which of the candidate plans will be chosen by the CBO.",
+      "defaultValue": "0.7"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.dp.star.filter",
+      "doc": "Applies star-join filter heuristics to cost based join enumeration.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.dp.threshold",
+      "doc": "The maximum number of joined nodes allowed in the dynamic programming algorithm.",
+      "defaultValue": "12"
+    },
+    {
+      "key": "spark.sql.cbo.joinReorder.enabled",
+      "doc": "Enables join reorder in CBO.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.planStats.enabled",
+      "doc": "When true, the logical plan will fetch row counts and column statistics from catalog.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cbo.starJoinFTRatio",
+      "doc": "Specifies the upper limit of the ratio between the largest fact tables for a star join to be considered. ",
+      "defaultValue": "0.9"
+    },
+    {
+      "key": "spark.sql.cbo.starSchemaDetection",
+      "doc": "When true, it enables join reordering based on star schema detection. ",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.charAsVarchar",
+      "doc": "When true, Spark replaces CHAR type with VARCHAR type in CREATE/REPLACE/ALTER TABLE commands, so that newly created/updated tables will not have CHAR type columns/fields. Existing tables with CHAR type columns/fields are not affected by this config.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.chunkBase64String.enabled",
+      "doc": "Whether to truncate string generated by the `Base64` function. When true, base64 strings generated by the base64 function are chunked into lines of at most 76 characters. When false, the base64 strings are not chunked.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.classic.shuffleDependency.fileCleanup.enabled",
+      "doc": "When enabled, shuffle files will be cleaned up at the end of classic SQL executions. Note that this cleanup may cause stage retries and regenerate shuffle files if the same dataframe reference is executed again.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.cli.print.header",
+      "doc": "When set to true, spark-sql CLI prints the names of the columns in query output.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.fastHashMap.capacityBit",
+      "doc": "Capacity for the max number of rows to be held in memory by the fast hash aggregate product operator. The bit is not for actual value, but the actual numBuckets is determined by loadFactor (e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.twolevel.enabled",
+      "doc": "Enable two-level aggregate hash map. When enabled, records will first be inserted/looked-up at a 1st-level, small, fast map, and then fallback to a 2nd-level, larger, slower map when 1st level is full or keys cannot be found. When disabled, records go directly to the 2nd level.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.twolevel.partialOnly",
+      "doc": "Enable two-level aggregate hash map for partial aggregate only, because final aggregate might get more distinct keys compared to partial aggregate. Overhead of looking up 1st-level map might dominate when having a lot of distinct keys.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.map.vectorized.enable",
+      "doc": "Enable vectorized aggregate hash map. This is for testing/benchmarking only.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.sortAggregate.enabled",
+      "doc": "When true, enable code-gen for sort aggregate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.aggregate.splitAggregateFunc.enabled",
+      "doc": "When true, the code generator would split aggregate code into individual methods instead of a single big method. This can be used to avoid oversized function that can miss the opportunity of JIT optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.broadcastCleanedSourceThreshold",
+      "doc": "A threshold (in string length) to determine if we should make the generated code a broadcast variable in whole stage codegen. To disable this, set the threshold to < 0; otherwise if the size is above the threshold, it'll use broadcast variable. Note that maximum string length allowed in Java is Integer.MAX_VALUE, so anything above it would be meaningless. The default value is set to -1 (disabled by default).",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.codegen.cache.maxEntries",
+      "doc": "When nonzero, enable caching of generated classes for operators and expressions. All jobs share the cache that can use up to the specified number for generated classes.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.codegen.comments",
+      "doc": "When true, put comment in the generated code. Since computing huge comments can be extremely expensive in certain cases, such as deeply-nested expressions which operate over inputs with wide schemas, default is false.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.codegen.factoryMode",
+      "doc": "This config determines the fallback behavior of several codegen generators during tests. `FALLBACK` means trying codegen first and then falling back to interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. `NO_CODEGEN` skips codegen and goes interpreted path always. Note that this configuration is only for the internal usage, and NOT supposed to be set by end users.",
+      "defaultValue": "FALLBACK"
+    },
+    {
+      "key": "spark.sql.codegen.fallback",
+      "doc": "When true, (whole stage) codegen could be temporary disabled for the part of query that fail to compile generated code",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.hugeMethodLimit",
+      "doc": "The maximum bytecode size of a single compiled Java function generated by whole-stage codegen. When the compiled function exceeds this threshold, the whole-stage codegen is deactivated for this subtree of the current query plan. The default value is 65535, which is the largest bytecode size possible for a valid Java method. When running on HotSpot, it may be preferable to set the value to 8000 to match HotSpot's implementation.",
+      "defaultValue": "65535"
+    },
+    {
+      "key": "spark.sql.codegen.join.buildSideOuterShuffledHashJoin.enabled",
+      "doc": "When true, enable code-gen for an OUTER shuffled hash join where outer side is the build side.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.existenceSortMergeJoin.enabled",
+      "doc": "When true, enable code-gen for Existence sort merge join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.fullOuterShuffledHashJoin.enabled",
+      "doc": "When true, enable code-gen for FULL OUTER shuffled hash join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.join.fullOuterSortMergeJoin.enabled",
+      "doc": "When true, enable code-gen for FULL OUTER sort merge join.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.logLevel",
+      "doc": "Configures the log level for logging of codegen. The value can be ERROR, WARN, INFO, DEBUG, TRACE.",
+      "defaultValue": "DEBUG"
+    },
+    {
+      "key": "spark.sql.codegen.logging.maxLines",
+      "doc": "The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.codegen.maxFields",
+      "doc": "The maximum number of fields (including nested fields) that will be supported before deactivating whole-stage codegen.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.codegen.methodSplitThreshold",
+      "doc": "The threshold of source-code splitting in the codegen. When the number of characters in a single Java function (without comment) exceeds the threshold, the function will be automatically split to multiple smaller ones. We cannot know how many bytecode will be generated, so use the code length as metric. When running on HotSpot, a function's bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not be too small, otherwise there will be many function calls.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.codegen.splitConsumeFuncByOperator",
+      "doc": "When true, whole stage codegen would put the logic of consuming rows of each physical operator into individual methods, instead of a single big method. This can be used to avoid oversized function that can miss the opportunity of JIT optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.useIdInClassName",
+      "doc": "When true, embed the (whole-stage) codegen stage ID into the class name of the generated class as a suffix",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.wholeStage",
+      "doc": "When true, the whole stage (of multiple operators) will be compiled into single java method.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.codegen.wholeStage.union.enabled",
+      "doc": "When both this conf and `spark.sql.codegen.wholeStage` are true, UnionExec participates in whole-stage codegen on its non-partitioning-aware path: the parent and all children fuse into a single WholeStageCodegenExec stage.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.codegen.wholeStage.union.maxChildren",
+      "doc": "Maximum number of UnionExec children eligible for whole-stage codegen fusion. Each child is emitted as its own helper method, so this conf bounds class-level costs of the fused stage (total bytecode size, constant pool growth, JIT compilation time) rather than the JVM per-method bytecode limit. Unions with more children fall back to per-child codegen stages. Only effective when `spark.sql.codegen.wholeStage.union.enabled` is true.",
+      "defaultValue": "64"
+    },
+    {
+      "key": "spark.sql.collation.allowInMapKeys",
+      "doc": "Allow for non-UTF8_BINARY collated strings inside of map's keys",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.collation.objectLevel.enabled",
+      "doc": "Object level collations feature is under development and its use should be done under this feature flag. The feature allows setting default collation for all underlying columns within that object, except the ones that were previously created.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.collation.schemaLevel.enabled",
+      "doc": "Schema level collations feature is under development and its use should be done under this feature flag. The feature allows setting default collation for all underlying objects within that schema, except the ones that were previously created.An object with an explicitly set collation will not inherit the collation from the schema.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.columnNameOfCorruptRecord",
+      "doc": "The name of internal column for storing raw/un-parsed JSON and CSV records that fail to parse.",
+      "defaultValue": "_corrupt_record"
+    },
+    {
+      "key": "spark.sql.columnVector.offheap.enabled",
+      "doc": "When true, use OffHeapColumnVector in ColumnarBatch.",
+      "fallback": "spark.memory.offHeap.enabled"
+    },
+    {
+      "key": "spark.sql.connect.enrichError.enabled",
+      "doc": "\nWhen true, it enriches errors with full exception messages and optionally server-side\nstacktrace on the client side via an additional RPC.\n",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.connect.serverStacktrace.enabled",
+      "doc": "When true, it sets the server-side stacktrace in the user-facing Spark exception.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.connect.shuffleDependency.fileCleanup.enabled",
+      "doc": "When enabled, shuffle files will be cleaned up at the end of Spark Connect SQL executions.",
+      "fallback": "spark.sql.shuffleDependency.fileCleanup.enabled"
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedSessions",
+      "doc": "The number of client sessions kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.connect.ui.retainedStatements",
+      "doc": "The number of statements kept in the Spark Connect UI history.",
+      "defaultValue": "200",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.constraintPropagation.enabled",
+      "doc": "When true, the query optimizer will infer and propagate data constraints in the query plan to optimize them. Constraint propagation can sometimes be computationally expensive for certain kinds of query plans (such as those with a large number of predicates and aliases) which might negatively impact overall runtime.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.crossJoin.enabled",
+      "doc": "When false, we will throw an error if a query contains a cartesian product without explicit CROSS JOIN syntax.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to CSV datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.parser.columnPruning.enabled",
+      "doc": "If it is set to true, column names of the requested schema are passed to CSV parser. Other column values can be ignored during parsing even if they are malformed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.csv.parser.inputBufferSize",
+      "doc": "If it is set, it configures the buffer size of CSV input during parsing. It is the same as inputBufferSize option in CSV which has a higher priority. Note that this is a workaround for the parsing library's regression, and this configuration is internal and supposed to be removed in the near future."
+    },
+    {
+      "key": "spark.sql.cteRecursionAnchorRowsLimitToConvertToLocalRelation",
+      "doc": "Maximum number of rows that the anchor in a recursive CTE can return for it to beconverted to a localRelation.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.cteRecursionLevelLimit",
+      "doc": "Maximum level of recursion that is allowed while executing a recursive CTE definition.If a query does not get exhausted before reaching this limit it fails. Use -1 for unlimited.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.cteRecursionRowLimit",
+      "doc": "Maximum number of rows that can be returned when executing a recursive CTE definition.If a query does not get exhausted before reaching this limit it fails. Use -1 for unlimited.",
+      "defaultValue": "1000000"
+    },
+    {
+      "key": "spark.sql.cteRelationDefMaxRows.enabled",
+      "doc": "When set to true, CTERelationDef.maxRows would output the correct value from the child plan. This is necessary for correct scalar subquery validation in the single-pass Analyzer.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.dataFrameQueryContext.enabled",
+      "doc": "Enable the DataFrame query context. This feature is enabled by default, but has a non-trivial performance overhead because of the stack trace collection.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.dataSource.alwaysCreateV2Predicate",
+      "doc": "When true, the v2 push-down framework always wraps the expression that returns boolean type with a v2 Predicate so that it can be pushed down.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.dataSource.skipAssertOnPredicatePushdown",
+      "doc": "Enable skipping assert when expression in not translated to predicate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.dataframeCache.logLevel",
+      "doc": "Configures the log level of Dataframe cache operations, including adding and removing entries from Dataframe cache, hit and miss on cache application. This log should only be used for debugging purposes and not in the production environment, since it generates a large amount of logs.",
+      "defaultValue": "TRACE"
+    },
+    {
+      "key": "spark.sql.datetime.java8API.enabled",
+      "doc": "If the configuration property is set to true, java.time.Instant and java.time.LocalDate classes of Java 8 API are used as external types for Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp and java.sql.Date are used for the same purpose.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.debug",
+      "doc": "Only used for internal debugging. Not all functions are supported when it is enabled.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.debug.maxToStringFields",
+      "doc": "Maximum number of fields of sequence-like entries can be converted to strings in debug output. Any elements beyond the limit will be dropped and replaced by a  \"... N more fields\" placeholder.",
+      "defaultValue": "25"
+    },
+    {
+      "key": "spark.sql.decimalOperations.allowPrecisionLoss",
+      "doc": "When true (default), establishing the result type of an arithmetic operation happens according to Hive behavior and SQL ANSI 2011 specification, i.e. rounding the decimal part of the result if an exact representation is not possible. Otherwise, NULL is returned in those cases, as previously.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultCacheStorageLevel",
+      "doc": "The default storage level of `dataset.cache()`, `catalog.cacheTable()` and sql query `CACHE TABLE t`.",
+      "defaultValue": "MEMORY_AND_DISK"
+    },
+    {
+      "key": "spark.sql.defaultCatalog",
+      "doc": "Name of the default catalog. This will be the current catalog if users have not explicitly set the current catalog yet.",
+      "defaultValue": "spark_catalog"
+    },
+    {
+      "key": "spark.sql.defaultColumn.allowedProviders",
+      "doc": "List of table providers wherein SQL commands are permitted to assign DEFAULT column values. Comma-separated list, whitespace ignored, case-insensitive. If an asterisk appears after any table provider in this list, any command may assign DEFAULT column except `ALTER TABLE ... ADD COLUMN`. Otherwise, if no asterisk appears, all commands are permitted. This is useful because in order for such `ALTER TABLE ... ADD COLUMN` commands to work, the target data source must include support for substituting in the provided values when the corresponding fields are not present in storage.",
+      "defaultValue": "csv,json,orc,parquet"
+    },
+    {
+      "key": "spark.sql.defaultColumn.enabled",
+      "doc": "When true, allow CREATE TABLE, REPLACE TABLE, and ALTER COLUMN statements to set or update default values for specific columns. Following INSERT, MERGE, and UPDATE statements may then omit these values and their values will be injected automatically instead.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultColumn.useNullsForMissingDefaultValues",
+      "doc": "When true, and DEFAULT columns are enabled, allow INSERT INTO commands with user-specified lists of fewer columns than the target table to behave as if they had specified DEFAULT for all remaining columns instead, in order.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.defaultPath",
+      "doc": "Default SQL PATH used when no SET PATH has been issued in the session; this is also the value to which `SET PATH = DEFAULT_PATH` expands. Accepts the full SET PATH grammar; an inner DEFAULT_PATH token resolves to the spark-builtin default ordering. The PATH keyword is not allowed in this conf value. When empty, the spark-builtin default ordering controlled by `spark.sql.functionResolution.sessionOrder` applies. Validated for syntax at set time; redundant entries are tolerated (lookup uses first-match resolution). The interactive SET PATH form still rejects static duplicates as a typo guard.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.defaultSizeInBytes",
+      "doc": "The default table size used in query planning. By default, it is set to Long.MaxValue which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. That is to say by default the optimizer will not choose to broadcast a table unless it knows for sure its size is small enough.",
+      "defaultValue": "9223372036854775807b"
+    },
+    {
+      "key": "spark.sql.defaultUrlStreamHandlerFactory.enabled",
+      "doc": "When true, register Hadoop's FsUrlStreamHandlerFactory to support ADD JAR against HDFS locations. It should be disabled when a different stream protocol handler should be registered to support a particular protocol type, or if Hadoop's FsUrlStreamHandlerFactory conflicts with other protocol types such as `http` or `https`. See also SPARK-25694 and HADOOP-14598.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.dropTableOnView.enabled",
+      "doc": "When true, DROP TABLE command will work on VIEW as well.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.enforceTypeCoercionBeforeUnionDeduplication.enabled",
+      "doc": "When set to true, we enforce type coercion to run before deduplication of UNION children outputs. Otherwise, order is relative to rule ordering.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.error.messageFormat",
+      "doc": "When PRETTY, the error message consists of textual representation of error class, message and query context. Stack traces are only shown for internal errors (SQLSTATE XX***). When DEBUG, the output is the same as PRETTY but stack traces are always included. The MINIMAL and STANDARD formats are pretty JSON formats where STANDARD includes an additional JSON field `message`. This configuration property influences on error messages of Thrift Server and SQL CLI while running queries.",
+      "defaultValue": "PRETTY"
+    },
+    {
+      "key": "spark.sql.event.truncate.length",
+      "doc": "Threshold of SQL length beyond which it will be truncated before adding to event. Defaults to no truncation. If set to 0, callsite will be logged instead.",
+      "defaultValue": "2147483647",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.exchange.reuse",
+      "doc": "When true, the planner will try to find out duplicated exchanges and re-use them.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.arrow.compression.codec",
+      "doc": "Compression codec used to compress Arrow IPC data when transferring data between JVM and Python processes (e.g., toPandas, toArrow). This can significantly reduce memory usage and network bandwidth when transferring large datasets. Supported codecs: 'none' (no compression), 'zstd' (Zstandard), 'lz4' (LZ4). Note that compression may add CPU overhead but can provide substantial memory savings especially for datasets with high compression ratios.",
+      "defaultValue": "none"
+    },
+    {
+      "key": "spark.sql.execution.arrow.compression.zstd.level",
+      "doc": "Compression level for Zstandard (zstd) codec when compressing Arrow IPC data. This config is only used when spark.sql.execution.arrow.compression.codec is set to 'zstd'. Negative values provide ultra-fast compression with lower compression ratios. Positive values provide normal to maximum compression, with higher values giving better compression but slower speed. The default value 3 provides a good balance between compression speed and compression ratio.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.execution.arrow.enabled",
+      "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.execution.arrow.pyspark.enabled' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.execution.arrow.fallback.enabled",
+      "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.fallback.enabled'.)",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Use 'spark.sql.execution.arrow.pyspark.fallback.enabled' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.execution.arrow.localRelationThreshold",
+      "doc": "When converting Arrow batches to Spark DataFrame, local collections are used in the driver side if the byte size of Arrow batches is smaller than this threshold. Otherwise, the Arrow batches are sent and deserialized to Spark internal rows in the executors.",
+      "defaultValue": "48MB"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxBytesPerBatch",
+      "doc": "When using Apache Arrow, limit the maximum bytes in each batch that can be written to a single ArrowRecordBatch in memory. Unlike 'spark.sql.execution.arrow.maxRecordsPerBatch', this configuration does not work for createDataFrame/toPandas with Arrow/pandas instances. See also spark.sql.execution.arrow.maxRecordsPerBatch. If both are set, each batch is created when any condition of both is met.",
+      "defaultValue": "64MB"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxBytesPerOutputBatch",
+      "doc": "When using Apache Arrow, limit the maximum bytes that can be output in a single ArrowRecordBatch to the downstream operator. If set to zero or negative there is no limit. Note that the complete ArrowRecordBatch is actually created but the number of bytes is limited when sending it to the downstream operator. This is used to avoid large batches being sent to the downstream operator including the columnar-based operator implemented by third-party libraries. Spark will try to create batches with the size equal or less than this value. Normally it should not happen, but if in extreme case that even one record is still larger than this value, Spark will create a batch with one record.",
+      "defaultValue": "-1b"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxRecordsPerBatch",
+      "doc": "When using Apache Arrow, limit the maximum number of records that can be written to a single ArrowRecordBatch in memory. If set to zero or negative there is no limit. See also spark.sql.execution.arrow.maxBytesPerBatch. If both are set, each batch is created when any condition of both is met.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.execution.arrow.maxRecordsPerOutputBatch",
+      "doc": "When using Apache Arrow, limit the maximum number of records that can be output in a single ArrowRecordBatch to the downstream operator. If set to zero or negative there is no limit. Note that the complete ArrowRecordBatch is actually created but the number of records is limited when sending it to the downstream operator. This is used to avoid large batches being sent to the downstream operator including the columnar-based operator implemented by third-party libraries.",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.enabled",
+      "doc": "When true, make use of Apache Arrow for columnar data transfers in PySpark. This optimization applies to: 1. pyspark.sql.DataFrame.toPandas. 2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame or a NumPy ndarray. The following data type is unsupported: ArrayType of TimestampType.",
+      "fallback": "spark.sql.execution.arrow.enabled"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.fallback.enabled",
+      "doc": "When true, optimizations enabled by 'spark.sql.execution.arrow.pyspark.enabled' will fallback automatically to non-optimized implementations if an error occurs.",
+      "fallback": "spark.sql.execution.arrow.fallback.enabled"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.selfDestruct.enabled",
+      "doc": "(Experimental) When true, make use of Apache Arrow's self-destruct and split-blocks options for columnar data transfers in PySpark, when converting from Arrow to Pandas. This reduces memory usage at the cost of some CPU time. This optimization applies to: pyspark.sql.DataFrame.toPandas when 'spark.sql.execution.arrow.pyspark.enabled' is set.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pyspark.validateSchema.enabled",
+      "doc": "When true, validate the schema of Arrow batches returned by mapInArrow, mapInPandas and DataSource against the expected schema to ensure that they are compatible.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.pythonUDF.columnarInput.enabled",
+      "doc": "When true, Arrow-based Python UDFs (pandas UDFs) can accept columnar input directly from upstream operators that produce Arrow-backed ColumnarBatch (e.g., DataSource V2 connectors), bypassing the ColumnarToRow and ArrowWriter conversion. This optimization reduces data transfer overhead between the JVM and Python worker processes.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.arrow.sparkr.enabled",
+      "doc": "When true, make use of Apache Arrow for columnar data transfers in SparkR. This optimization applies to: 1. createDataFrame when its input is an R DataFrame 2. collect 3. dapply 4. gapply The following data types are unsupported: FloatType, BinaryType, ArrayType, StructType and MapType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.arrow.transformWithStateInPySpark.maxStateRecordsPerBatch",
+      "doc": "When using TransformWithState in PySpark (both Python Row and Pandas), limit the maximum number of state records that can be written to a single ArrowRecordBatch in memory.",
+      "defaultValue": "10000",
+      "alternatives": [
+        "spark.sql.execution.arrow.transformWithStateInPandas.maxRecordsPerBatch"
+      ]
+    },
+    {
+      "key": "spark.sql.execution.arrow.useLargeVarTypes",
+      "doc": "When using Apache Arrow, use large variable width vectors for string and binary types. Regular string and binary types have a 2GiB limit for a column in a single record batch. Large variable types remove this limitation at the cost of higher memory usage per value.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.broadcastHashJoin.outputPartitioningExpandLimit",
+      "doc": "The maximum number of partitionings that a HashPartitioning can be expanded to. This configuration is applicable only for BroadcastHashJoin inner joins and can be set to '0' to disable this feature.",
+      "defaultValue": "8"
+    },
+    {
+      "key": "spark.sql.execution.datasources.hadoopLineRecordReader.enabled",
+      "doc": "Enable the imported Hadoop's LineRecordReader. This was imported and renamed to HadoopLineRecordReader to add support for compression option and other future codecs like ZSTD, etc. Setting the conf to false will use the LineRecordReader class from the hadoop jar instead of the imported one.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.fastFailOnFileFormatOutput",
+      "doc": "Whether to fast fail task execution when writing output to FileFormat datasource. If this is enabled, in `FileFormatWriter` we will catch `FileAlreadyExistsException` and fast fail output task without further task retry. Only enabling this if you know the `FileAlreadyExistsException` of the output task is unrecoverable, i.e., further task attempts won't be able to success. If the `FileAlreadyExistsException` might be recoverable, you should keep this as disabled and let Spark to retry output tasks. This is disabled by default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.interruptOnCancel",
+      "doc": "When true, all running tasks will be interrupted if one cancels a query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pandas.convertToArrowArraySafely",
+      "doc": "When true, Arrow will perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. When false, disabling Arrow's type check and do type conversions anyway. This config only works for Arrow 0.11.0+.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pandas.inferPandasDictAsMap",
+      "doc": "When true, spark.createDataFrame will infer dict from Pandas DataFrame as a MapType. When false, spark.createDataFrame infers dict from Pandas DataFrame as a StructType which is default inferring from PyArrow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pandas.respectSessionTimeZone",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "The non-default behavior is considered as a bug, see SPARK-22395. The config was deprecated since Spark 2.3."
+      }
+    },
+    {
+      "key": "spark.sql.execution.pandas.structHandlingMode",
+      "doc": "The conversion mode of struct type when creating pandas DataFrame. When \"legacy\", 1. when Arrow optimization is disabled, convert to Row object, 2. when Arrow optimization is enabled, convert to dict or raise an Exception if there are duplicated nested field names. When \"row\", convert to Row object regardless of Arrow optimization. When \"dict\", convert to dict and use suffixed key names, e.g., a_0, a_1, if there are duplicated nested field names, regardless of Arrow optimization.",
+      "defaultValue": "legacy"
+    },
+    {
+      "key": "spark.sql.execution.pandas.udf.buffer.size",
+      "doc": "Same as `spark.buffer.size` but only applies to Pandas UDF executions. If it is not set, the fallback is `spark.buffer.size`. Note that Pandas execution requires more than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and pipelined; however, it might degrade performance. See SPARK-27870.",
+      "fallback": "spark.buffer.size"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.binaryAsBytes",
+      "doc": "When true, BinaryType is consistently mapped to bytes in PySpark. When false, restores the PySpark behavior before 4.1.0. Before 4.1.0, BinaryType is mapped to bytearray for regular UDF and UDTF without Arrow optimization, DataFrame APIs (both Spark Classic and Spark Connect), and data sources; BinaryType is mapped to bytes for Arrow-optimized UDF and UDTF with legacy pandas conversion.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.python",
+      "doc": "Python binary executable to use for PySpark in executors when running Python UDF, pandas UDF and pandas function APIs. If not set, it falls back to 'spark.pyspark.python' by default."
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.daemonKillWorkerOnFlushFailure",
+      "doc": "Same as spark.python.daemon.killWorkerOnFlushFailure for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.daemon.killWorkerOnFlushFailure"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.faulthandler.enabled",
+      "doc": "Same as spark.python.worker.faulthandler.enabled for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.worker.faulthandler.enabled"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.hideTraceback.enabled",
+      "doc": "When true, only show the message of the exception from Python UDFs, hiding the stack trace. If this is enabled, simplifiedTraceback has no effect.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.idleTimeoutSeconds",
+      "doc": "Same as spark.python.worker.idleTimeoutSeconds for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.worker.idleTimeoutSeconds"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.killOnIdleTimeout",
+      "doc": "Same as spark.python.worker.killOnIdleTimeout for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.worker.killOnIdleTimeout"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled",
+      "doc": "When true, the traceback from Python UDFs is simplified. It hides the Python worker, (de)serialization, etc from PySpark in tracebacks, and only shows the exception messages from UDFs. Note that this works only with CPython 3.7+.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pyspark.udf.tracebackDumpIntervalSeconds",
+      "doc": "Same as spark.python.worker.tracebackDumpIntervalSeconds for Python execution with DataFrame and SQL. It can change during runtime.",
+      "fallback": "spark.python.worker.tracebackDumpIntervalSeconds"
+    },
+    {
+      "key": "spark.sql.execution.python.udf.buffer.size",
+      "doc": "Same as `spark.buffer.size` but only applies to Python UDF executions. If it is not set, the fallback is `spark.buffer.size`.",
+      "fallback": "spark.buffer.size"
+    },
+    {
+      "key": "spark.sql.execution.python.udf.maxRecordsPerBatch",
+      "doc": "When using Python UDFs, limit the maximum number of records that can be batched for serialization/deserialization.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.arrow.concurrency.level",
+      "doc": "The level of concurrency to execute Arrow-optimized Python UDF. This can be useful if Python UDFs use I/O intensively."
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.arrow.enabled",
+      "doc": "Enable Arrow optimization in regular Python UDFs. This optimization can only be enabled when the given function takes at least one argument.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.arrow.legacy.fallbackOnUDT",
+      "doc": "When true, Arrow-optimized Python UDF will fallback to the regular UDF when its input or output is UDT.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.pandas.intToDecimalCoercionEnabled",
+      "doc": "When true, convert int to Decimal python objects before converting Pandas.Series to Arrow array during serialization.Disabled by default, impacts performance.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDF.pandas.preferIntExtensionDtype",
+      "doc": "When true, convert integers to Pandas ExtensionDtype (e.g. pandas.Int64Dtype) for Pandas UDF execution. Otherwise, depends on the behavior of pyarrow.Array.to_pandas on each input arrow batch.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.pythonUDTF.arrow.enabled",
+      "doc": "Enable Arrow optimization for Python UDTFs.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.rangeExchange.sampleSizePerPartition",
+      "doc": "Number of points to sample per partition in order to determine the range boundaries for range partitioning, typically used in global sorting (without limit).",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.execution.removeRedundantProjects",
+      "doc": "Whether to remove redundant project exec node based on children's output and ordering requirement.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.removeRedundantSorts",
+      "doc": "Whether to remove redundant physical sort node",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.replaceHashWithSortAgg",
+      "doc": "Whether to replace hash aggregate node with sort aggregate based on children's ordering",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.execution.reuseSubquery",
+      "doc": "When true, the planner will try to find out duplicated subqueries and re-use them.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.sortBeforeRepartition",
+      "doc": "When perform a repartition following a shuffle, the output row ordering would be nondeterministic. If some downstream stages fail and some tasks of the repartition stage retry, these tasks may generate different data, and that can lead to correctness issues. Turn on this config to insert a local sort before actually doing repartition to generate consistent repartition results. The performance of repartition() may go down since we insert extra local sort before it.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.topKSortFallbackThreshold",
+      "doc": "In SQL queries with a SORT followed by a LIMIT like 'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort in memory, otherwise do a global sort which spills to disk if necessary.",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.execution.useObjectHashAggregateExec",
+      "doc": "Decides if we use ObjectHashAggregateExec",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.execution.usePartitionEvaluator",
+      "doc": "When true, use PartitionEvaluator to execute SQL operators.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.expressionTreeChangeLog.level",
+      "doc": "Configures the log level for logging the change from the unresolved expression tree to the resolved expression tree in the single-pass bottom-up Resolver. The value can be ERROR, WARN, INFO, DEBUG, TRACE.",
+      "defaultValue": "TRACE"
+    },
+    {
+      "key": "spark.sql.extendedExplainProviders",
+      "doc": "A comma-separated list of classes that implement the org.apache.spark.sql.ExtendedExplainGenerator trait. If provided, Spark will print extended plan information from the providers in explain plan and in the UI"
+    },
+    {
+      "key": "spark.sql.extensions",
+      "doc": "A comma-separated list of classes that implement Function1[SparkSessionExtensions, Unit] used to configure Spark Session extensions. The classes must have a no-args constructor. If multiple extensions are specified, they are applied in the specified order. For the case of rules and planner strategies, they are applied in the specified order. For the case of parsers, the last parser is used and each parser can delegate to its predecessor. For the case of function name conflicts, the last registered function name is used.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.extensions.test.loadFromCp",
+      "doc": "Flag that determines if we should load extensions from the classpath using the SparkSessionExtensionsProvider mechanism. This is a test only flag.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.fileSource.insert.enforceNotNull",
+      "doc": "When true, Spark enforces NOT NULL constraints when inserting data into file-based data source tables (e.g., Parquet, ORC, JSON), consistent with the behavior for other data sources and V2 catalog tables. When false (default), null values are silently accepted into NOT NULL columns.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.ignoreCorruptFiles",
+      "doc": "Whether to ignore corrupt files. If true, the Spark jobs will continue to run when encountering corrupted files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.ignoreInvalidPartitionPaths",
+      "doc": "Whether to ignore invalid partition paths that do not match =. When the option is enabled, table with two partition directories 'table/invalid' and 'table/col=1' will only load the latter directory and ignore the invalid partition",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.ignoreMissingFiles",
+      "doc": "Whether to ignore missing files. If true, the Spark jobs will continue to run when encountering missing files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.files.maxPartitionBytes",
+      "doc": "The maximum number of bytes to pack into a single partition when reading files. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "128MB"
+    },
+    {
+      "key": "spark.sql.files.maxPartitionNum",
+      "doc": "The suggested (not guaranteed) maximum number of split file partitions. If it is set, Spark will rescale each partition to make the number of partitions is close to this value if the initial number of partitions exceeds this value. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC."
+    },
+    {
+      "key": "spark.sql.files.maxRecordsPerFile",
+      "doc": "Maximum number of records to write out to a single file. If this value is zero or negative, there is no limit.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.files.minPartitionNum",
+      "doc": "The suggested (not guaranteed) minimum number of split file partitions. If not set, the default value is `spark.sql.leafNodeDefaultParallelism`. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC."
+    },
+    {
+      "key": "spark.sql.files.openCostInBytes",
+      "doc": "The estimated cost to open a file, measured by the number of bytes could be scanned in the same time. This is used when putting multiple files into a partition. It's better to over estimated, then the partitions with small files will be faster than partitions with bigger files (which is scheduled first). This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "4MB"
+    },
+    {
+      "key": "spark.sql.files.supportSecondOffsetFormat",
+      "doc": "When set to true, datetime formatter used for csv, json and xml will support zone offsets that have seconds in it. e.g. LA timezone offset prior to 1883 was -07:52:58. When this flag is not set we lose seconds information.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.filesourceTableRelationCacheSize",
+      "doc": "The maximum size of the cache that maps qualified table names to table relation plans.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.fromJsonForceNullableSchema",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent errors like SPARK-23173 for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.function.concatBinaryAsString",
+      "doc": "When this option is set to false and all inputs are binary, `functions.concat` returns an output as binary. Otherwise, it returns as a string.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.function.eltOutputAsString",
+      "doc": "When this option is set to false and all inputs are binary, `elt` returns an output as binary. Otherwise, it returns as a string.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.function.protobufExtensions.enabled",
+      "doc": "When true, the from_protobuf and to_protobuf operators will support proto2 extensions when a binary file descriptor set is provided. This property will have no effect for the overloads taking a Java class name instead of a file descriptor set.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.functionResolution.sessionOrder",
+      "doc": "Order of the session (temporary) function namespace when resolving unqualified function names. Valid values: 'first', 'second', 'last'. 'first': session -> builtin -> persistent (SQL temp creation blocked if conflicts with builtin). 'second': builtin -> session -> persistent (default). 'last': builtin -> persistent (current schema) -> session.",
+      "defaultValue": "second"
+    },
+    {
+      "key": "spark.sql.geospatial.enabled",
+      "doc": "When true, enables geospatial types (GEOGRAPHY/GEOMETRY) and ST functions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.globalTempDatabase",
+      "doc": "",
+      "defaultValue": "global_temp",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.groupByAliases",
+      "doc": "When true, aliases in a select list can be used in group by clauses. When false, an analysis exception is thrown in the case.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.groupByOrdinal",
+      "doc": "When true, the ordinal numbers in group by clauses are treated as the position in the select list. When false, the ordinal numbers are ignored.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.advancedPartitionPredicatePushdown.enabled",
+      "doc": "When true, advanced partition predicate pushdown into Hive metastore is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.caseSensitiveInferenceMode",
+      "doc": "Sets the action to take when a case-sensitive schema cannot be read from a Hive Serde table's properties when reading the table with Spark native data sources. Valid options include INFER_AND_SAVE (infer the case-sensitive schema from the underlying data files and write it back to the table properties), INFER_ONLY (infer the schema but don't attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback to using the case-insensitive metastore schema instead of inferring).",
+      "defaultValue": "NEVER_INFER"
+    },
+    {
+      "key": "spark.sql.hive.convertCTAS",
+      "doc": "When true, a table created by a Hive CTAS statement (no USING clause) without specifying any storage property will be converted to a data source table, using the data source set by spark.sql.sources.default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.dropPartitionByName.enabled",
+      "doc": "When true, Spark will get partition name rather than partition object to drop partition, which can improve the performance of drop partition.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.filesourcePartitionFileCacheSize",
+      "doc": "When nonzero, enable caching of partition file metadata in memory. All tables share a cache that can use up to specified num bytes for file metadata. This conf only has an effect when hive filesource partition management is enabled.",
+      "defaultValue": "262144000"
+    },
+    {
+      "key": "spark.sql.hive.gatherFastStats",
+      "doc": "When true, fast stats (number of files and total size of all files) will be gathered in parallel while repairing table partitions to avoid the sequential listing in Hive metastore.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.manageFilesourcePartitions",
+      "doc": "When true, enable metastore partition management for file source tables as well. This includes both datasource and converted Hive tables. When partition management is enabled, datasource tables store partition in the Hive metastore, and use the metastore to prune partitions during query planning when spark.sql.hive.metastorePartitionPruning is set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruning",
+      "doc": "When true, some predicates will be pushed down into the Hive metastore so that unmatching partitions can be eliminated earlier.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningFallbackOnException",
+      "doc": "Whether to fallback to get all partitions from Hive metastore and perform partition pruning on Spark client side, when encountering MetaException from the metastore. Note that Spark query performance may degrade if this is enabled and there are many partitions to be listed. If this is disabled, Spark will fail the query instead.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningFastFallback",
+      "doc": "When this config is enabled, if the predicates are not supported by Hive or Spark does fallback due to encountering MetaException from the metastore, Spark will instead prune partitions by getting the partition names first and then evaluating the filter expressions on the client side. Note that the predicates with TimeZoneAwareExpression is not supported.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.hive.metastorePartitionPruningInSetThreshold",
+      "doc": "The threshold of set size for InSet predicate when pruning partitions through Hive Metastore. When the set size exceeds the threshold, we rewrite the InSet predicate to be greater than or equal to the minimum value in set and less than or equal to the maximum value in set. Larger values may cause Hive Metastore stack overflow. But for InSet inside Not with values exceeding the threshold, we won't push it to Hive Metastore.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.hive.tablePropertyLengthThreshold",
+      "doc": "The maximum length allowed in a single cell when storing Spark-specific information in Hive's metastore as table properties. Currently it covers 2 things: the schema's JSON string, the histogram of column statistics."
+    },
+    {
+      "key": "spark.sql.hive.thriftServer.singleSession",
+      "doc": "When set to true, Hive Thrift server is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.hive.verifyPartitionPath",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "This config was replaced by 'spark.sql.files.ignoreMissingFiles'."
+      }
+    },
+    {
+      "key": "spark.sql.icu.caseMappings.enabled",
+      "doc": "When enabled we use the ICU library (instead of the JVM) to implement case mappings for strings under UTF8_BINARY collation.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.batchSize",
+      "doc": "Controls the size of batches for columnar caching.  Larger batch sizes can improve memory utilization and compression, but risk OOMs when caching data.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.compressed",
+      "doc": "When set to true Spark SQL will automatically select a compression codec for each column based on statistics of the data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.enableVectorizedReader",
+      "doc": "Enables vectorized reader for columnar caching.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.hugeVectorReserveRatio",
+      "doc": "When spark.sql.inMemoryColumnarStorage.hugeVectorThreshold <= 0 or the required memory is smaller than spark.sql.inMemoryColumnarStorage.hugeVectorThreshold, spark reserves required memory * 2 memory; otherwise, spark reserves required memory * this ratio memory, and will release this column vector memory before reading the next batch rows.",
+      "defaultValue": "1.2"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.hugeVectorThreshold",
+      "doc": "When the required memory is larger than this, spark reserves required memory * spark.sql.inMemoryColumnarStorage.hugeVectorReserveRatio memory next time and release this column vector memory before reading the next batch rows. -1 means disabling the optimization.",
+      "defaultValue": "-1b"
+    },
+    {
+      "key": "spark.sql.inMemoryColumnarStorage.partitionPruning",
+      "doc": "When true, enable partition pruning for in-memory columnar tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.inMemoryTableScanStatistics.enable",
+      "doc": "When true, enable in-memory table scan accumulators.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.insertIntoReplaceOn.enabled",
+      "doc": "Enable the SQL syntax INSERT INTO ... REPLACE ON (...). The command atomically inserts new rows into a table after deleting all existing rows that match the new rows according to the specified matching condition. The inserted rows are specified by a VALUES expression or the result of a query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.insertIntoReplaceOnByName.enabled",
+      "doc": "Enable the SQL syntax INSERT INTO ... BY NAME REPLACE ON. Allows using the BY NAME clause with INSERT INTO REPLACE ON.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.insertIntoReplaceUsing.enabled",
+      "doc": "Enable the SQL syntax INSERT INTO ... REPLACE USING (...). The command atomically inserts new rows into a table after deleting all existing rows that match the new rows according to the key columns specified in the statement. The inserted rows are specified by a VALUES expression or the result of a query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.insertIntoReplaceUsingByName.enabled",
+      "doc": "Enable the SQL syntax INSERT INTO ... BY NAME REPLACE USING (...). Allows using the BY NAME clause with INSERT INTO REPLACE USING.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.insertNestedTypeCoercion.enabled",
+      "doc": "If enabled, allow INSERT INTO WITH SCHEMA EVOLUTION to fill missing nested struct fields with null when the source has fewer nested fields than the target table. Also relaxes by-position column-count enforcement so trailing missing top-level columns are filled with their default value (or null). This is experimental and the semantics may change.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.join.preferSortMergeJoin",
+      "doc": "When true, prefer sort merge join over shuffled hash join. Sort merge join consumes less memory than shuffled hash join and it works efficiently when both join tables are large. On the other hand, shuffled hash join can improve performance (e.g., of full outer joins) when one of join tables is much smaller.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.enableExactStringParsing",
+      "doc": "When set to true, string columns extracted from JSON objects will be extracted exactly as they appear in the input string, with no changes",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.enablePartialResults",
+      "doc": "When set to true, enables partial results for structs, maps, and arrays in JSON when one or more fields do not match the schema",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to JSON datasource.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.json.useUnsafeRow",
+      "doc": "When set to true, use UnsafeRow to represent struct result in the JSON parser. It can be overwritten by the JSON option `useUnsafeRow`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.jsonGenerator.ignoreNullFields",
+      "doc": "Whether to ignore null fields when generating JSON objects in JSON data source and JSON functions such as to_json. If false, it generates null for null fields in JSON objects.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.jsonGenerator.writeNullIfWithDefaultValue",
+      "doc": "When true, when writing NULL values to columns of JSON tables with explicit DEFAULT values using INSERT, UPDATE, or MERGE commands, never skip writing the NULL values to storage, overriding spark.sql.jsonGenerator.ignoreNullFields or the ignoreNullFields option. This can be useful to enforce that inserted NULL values are present in storage to differentiate from missing data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lateralColumnAlias.enableImplicitResolution",
+      "doc": "Enable resolving implicit lateral column alias defined in the same SELECT list. For example, with this conf turned on, for query `SELECT 1 AS a, a + 1` the `a` in `a + 1` can be resolved as the previously defined `1 AS a`. But note that table column has higher resolution priority than the lateral column alias.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lazySetOperatorOutput.enabled",
+      "doc": "When set to true, Except/Intersect/Union operator's output will be a lazy val. It is a performance optimization for querires with a large number of stacked set operators. This is because of rules like WidenSetOperationTypes that traverse the logical plan tree and call output on each Except/Intersect/Union node. Such traversal has quadratic complexity: O(number_of_nodes * (1 + 2 + 3  + ... + number_of_nodes)).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.leafNodeDefaultParallelism",
+      "doc": "The default parallelism of Spark SQL leaf nodes that produce data, such as the file scan node, the local data scan node, the range node, etc. The default value of this config is 'SparkContext#defaultParallelism'."
+    },
+    {
+      "key": "spark.sql.legacy.addSingleFileInAddFile",
+      "doc": "When true, only a single file can be added using ADD FILE. If false, then users can add directory by passing directory path to ADD FILE.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowAutoGeneratedAliasForView",
+      "doc": "When true, it's allowed to use a input query without explicit alias when creating a permanent view.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent loss of user data for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.allowEmptySchemaWrite",
+      "doc": "When this option is set to true, validation of empty or empty nested schemas that occurs when writing into a FileFormat based data source does not happen.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowHashOnMapType",
+      "doc": "When set to true, hash expressions can be applied on elements of MapType. Otherwise, an analysis exception will be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNegativeScaleOfDecimal",
+      "doc": "When set to true, negative scale of Decimal type is allowed. For example, the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is Decimal(11, 0) in non legacy mode.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNonEmptyLocationInCTAS",
+      "doc": "When false, CTAS with LOCATION throws an analysis exception if the location is not empty.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowNullComparisonResultInArraySort",
+      "doc": "When set to false, `array_sort` function throws an error if the comparator function returns null. If set to true, it restores the legacy behavior that handles null as zero (equal).",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowParameterlessCount",
+      "doc": "When true, the SQL function 'count' is allowed to take no parameters.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowSessionVariableInPersistedView",
+      "doc": "When set to true, variables can be found under identifiers in a view query. Throw otherwise.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowStarWithSingleTableIdentifierInCount",
+      "doc": "When true, the SQL function 'count' is allowed to take single 'tblName.*' as parameter",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowTempViewCreationWithMultipleNameparts",
+      "doc": "When true, temp view creation Dataset APIs will allow the view creation even if the view name is multiple name parts. The extra name parts will be dropped during the view creation",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowUdfParameterToShadowParameterlessFunction",
+      "doc": "When true (legacy behavior), a SQL UDF parameter alias shadows a parameterless built-in function (current_user, current_date, current_time, current_timestamp, user, session_user, grouping__id) of the same name. When false (the default), the parameterless built-in function takes precedence, matching the documented name resolution rules.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowUntypedScalaUDF",
+      "doc": "When set to true, user is allowed to use org.apache.spark.sql.functions. udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.allowZeroIndexInFormatString",
+      "doc": "When false, the `strfmt` in `format_string(strfmt, obj, ...)` and `printf(strfmt, obj, ...)` will no longer support to use \"0$\" to specify the first argument, the first argument should always reference by \"1$\" when use argument index to indicating the position of the argument in the argument list. This config will be removed in the future releases.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Increase indexes by 1 in `strfmt` of the `format_string` function. Refer to the first argument by \"1$\"."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.avro.allowIncompatibleSchema",
+      "doc": "When set to false, if types in Avro are encoded in the same format, but the type in the Avro schema explicitly says that the data types are different, reject reading the data type in the format to avoid returning incorrect results. When set to true, it restores the legacy behavior of allow reading the data in the format, which may return incorrect results.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.avro.datetimeRebaseModeInRead",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.avro.datetimeRebaseModeInRead' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.avro.datetimeRebaseModeInWrite",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.avro.datetimeRebaseModeInWrite' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.bangEqualsNot",
+      "doc": "When set to true, '!' is a lexical equivalent for 'NOT'. That is '!' can be used outside of the documented prefix usage in a logical expression. Examples are: `expr ! IN (1, 2)` and `expr ! BETWEEN 1 AND 2`, but also `IF ! EXISTS`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.blockCreateTempTableUsingProvider",
+      "doc": "If enabled, we fail legacy CREATE TEMPORARY TABLE ... USING provider during parsing.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.bucketedTableScan.outputOrdering",
+      "doc": "When true, the bucketed table scan will list files during planning to figure out the output ordering, which is expensive and may make the planning quite slow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.castComplexTypesToString.enabled",
+      "doc": "When true, maps and structs are wrapped by [] in casting to strings, and NULL elements of structs/maps/arrays will be omitted while converting to strings. Otherwise, if this is false, which is the default, maps and structs are wrapped by {}, and NULL elements will be converted to \"null\".",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.charVarcharAsString",
+      "doc": "When true, Spark treats CHAR/VARCHAR type the same as STRING type, which is the behavior of Spark 3.0 and earlier. This means no length check for CHAR/VARCHAR type and no padding for CHAR type when writing data to the table.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.codingErrorAction",
+      "doc": "When set to true, encode/decode functions replace unmappable characters with mojibake instead of reporting coding errors.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.collationAwareHashFunctions",
+      "doc": "Enables collation aware hashing (legacy behavior) for collated strings in Murmur3Hash and XxHash64 user-facing expressions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.compareDateTimestampInTimestamp",
+      "doc": "(Removed)",
+      "defaultValue": "true",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "It was removed to prevent errors like SPARK-23549 for non-default value."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.consecutiveStringLiterals.enabled",
+      "doc": "When true, consecutive string literals separated by double quotes (e.g. 'a''b') will be parsed as concatenated strings. This preserves pre-Spark 4.0 behavior where'a''b' would be parsed as 'ab' instead of 'a'b'.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.createEmptyCollectionUsingStringType",
+      "doc": "When set to true, Spark returns an empty collection with `StringType` as element type if the `array`/`map` function is called without any parameters. Otherwise, Spark returns an empty collection with `NullType` as element type.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.createHiveTableByDefault",
+      "doc": "When set to true, CREATE TABLE syntax without USING or STORED AS will use Hive instead of the value of spark.sql.sources.default as the table provider.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.csv.enableDateTimeParsingFallback",
+      "doc": "When true, enable legacy date/time parsing fallback in CSV"
+    },
+    {
+      "key": "spark.sql.legacy.cteDuplicateAttributeNames",
+      "doc": "When true, CTE references sometimes (e.g., self-joins) may incorrectly unify columnnames that differ only in casing (e.g., COLNAME and colname become the same). This isthe legacy behavior. When false, column name casing is preserved correctly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.ctePrecedencePolicy",
+      "doc": "When LEGACY, outer CTE definitions takes precedence over inner definitions. If set to EXCEPTION, AnalysisException is thrown while name conflict is detected in nested CTE. The default is CORRECTED, inner CTE definitions take precedence. This config will be removed in future versions and CORRECTED will be the only behavior.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.legacy.dataFrameWriterV2IgnorePathOption",
+      "doc": "When set to true, DataFrameWriterV2 ignores the 'path' option and always write data to the default table location.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue",
+      "doc": "When set to true, the key attribute resulted from running `Dataset.groupByKey` for non-struct key type, will be named as `value`, following the behavior of Spark version 2.4 and earlier.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.db2.booleanMapping.enabled",
+      "doc": "When true, BooleanType maps to CHAR(1) in DB2; otherwise, BOOLEAN",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.db2.numericMapping.enabled",
+      "doc": "When true, SMALLINT maps to IntegerType in DB2; otherwise, ShortType",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.decimal.retainFractionDigitsOnTruncate",
+      "doc": "When set to true, we will try to retain the fraction digits first rather than integral digits as prior Spark 4.0, when getting a least common type between decimal types, and the result decimal precision exceeds the max precision.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.disableMapKeyNormalization",
+      "doc": "Disables key normalization when creating a map with `ArrayBasedMapBuilder`. When set to `true` it will prevent key normalization when building a map, which will allow for values such as `-0.0` and `0.0` to be present as distinct keys.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.doLooseUpcast",
+      "doc": "When true, the upcast will be loose and allows string to atomic types.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.duplicateBetweenInput",
+      "doc": "When true, we use legacy between implementation. This is a flag that fixes a problem introduced by a between optimization, see ticket SPARK-49063.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.earlyEvalCurrentTime",
+      "doc": "When set to true, evaluation and constant folding will happen for now() and current_timestamp() expressions before finish analysis phase. This flag will allow a bit more liberal syntax but it will sacrifice correctness - Results of now() and current_timestamp() can be different for different operations in a single query.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.emptyCurrentDBInCli",
+      "doc": "When false, spark-sql CLI prints the current database in prompt.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName",
+      "doc": "When true, columns will be looked up by name if labeled with a string and fallback to use position if not. When false, a grouped map Pandas UDF will assign columns from the returned Pandas DataFrame based on position, regardless of column label type. This configuration will be deprecated in future releases.",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "2.4",
+        "comment": "The config allows to switch to the behaviour before Spark 2.4 and will be removed in the future releases."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled",
+      "doc": "When true and spark.sql.execution.pythonUDF.arrow.enabled is enabled, matches the default Arrow Python UDF behavior before 4.1.0. With this behavior, extrapandas conversion happens during (de)serialization between JVM and Python workers. This matters especially when the produced output has a schema different from specified schema, resulting in a different type coercion.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.execution.pythonUDTF.pandas.conversion.enabled",
+      "doc": "When true and spark.sql.execution.pythonUDTF.arrow.enabled is enabled, extra pandas conversion happens during (de)serialization between JVM and Python workers. This matters especially when the produced output has a schema different from specified schema, resulting in a different type coercion.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.exponentLiteralAsDecimal.enabled",
+      "doc": "When set to true, a literal with an exponent (e.g. 1E-30) would be parsed as Decimal rather than Double.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.extraOptionsBehavior.enabled",
+      "doc": "When true, the extra options will be ignored for DataFrameReader.table(). If set it to false, which is the default, Spark will check if the extra options have the same key, but the value is different with the table serde properties. If the check passes, the extra options will be merged with the serde properties as the scan options. Otherwise, an exception will be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.followThreeValuedLogicInArrayExists",
+      "doc": "When true, the ArrayExists will follow the three-valued boolean logic.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.fromDayTimeString.enabled",
+      "doc": "When true, the `from` bound is not taken into account in conversion of a day-time string to an interval, and the `to` bound is used to skip all interval units out of the specified range. If it is set to `false`, `ParseException` is thrown if the input does not match to the pattern defined by `from` and `to`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.groupingIdWithAppendedUserGroupBy",
+      "doc": "When true, grouping_id() returns values based on grouping set columns plus user-given group-by expressions order like Spark 3.2.0, 3.2.1, 3.2.2, and 3.3.0.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.histogramNumericPropagateInputType",
+      "doc": "The histogram_numeric function computes a histogram on numeric 'expr' using nb bins. The return value is an array of (x,y) pairs representing the centers of the histogram's bins. If this config is set to true, the output type of the 'x' field in the return value is propagated from the input value consumed in the aggregate function. Otherwise, 'x' always has double type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.identifierClause",
+      "doc": "When set to false, IDENTIFIER('literal') is resolved to an identifier at parse time anywhere identifiers can occur. When set to true, only the legacy  IDENTIFIER(constantExpr) clause is allowed, which evaluates the expression at analysis  and is limited to a narrow subset of scenarios.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.inSubqueryNullability",
+      "doc": "When set to false, IN subquery nullability is correctly calculated based on both the left and right sides of the IN. When set to true, restores the legacy behavior that does not check the right side's nullability.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.inlineCTEInCommands",
+      "doc": "If true, always inline the CTE relations for the queries in commands. This is the legacy behavior which may produce incorrect results because Spark may evaluate a CTE relation more than once, even if it's nondeterministic.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.integerGroupingId",
+      "doc": "When true, grouping_id() returns int values instead of long values.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.interval.enabled",
+      "doc": "When set to true, Spark SQL uses the mixed legacy interval type `CalendarIntervalType` instead of the ANSI compliant interval types `YearMonthIntervalType` and `DayTimeIntervalType`. For instance, the date subtraction expression returns `CalendarIntervalType` when the SQL config is set to `true` otherwise an ANSI interval.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.javaCharsets",
+      "doc": "When set to true, the functions like `encode()` can use charsets from JDK while encoding or decoding string values. If it is false, such functions support only one of the charsets: 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.json.allowEmptyString.enabled",
+      "doc": "When set to true, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.json.enableDateTimeParsingFallback",
+      "doc": "When true, enable legacy date/time parsing fallback in JSON"
+    },
+    {
+      "key": "spark.sql.legacy.keepCommandOutputSchema",
+      "doc": "When true, Spark will keep the output schema of commands such as SHOW DATABASES unchanged.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.keepPartitionSpecAsStringLiteral",
+      "doc": "If it is set to true, `PARTITION(col=05)` is parsed as a string literal of its text representation, e.g., string '05', when the partition column is string type. Otherwise, it is always parsed as a numeric literal in the partition spec.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.literal.pickMinimumPrecision",
+      "doc": "When integral literal is used in decimal operations, pick a minimum precision required by the literal if this config is true, to make the resulting precision and/or scale smaller. This can reduce the possibility of precision lose and/or overflow.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.lpadRpadAlwaysReturnString",
+      "doc": "When set to false, when the first argument and the optional padding pattern is a byte sequence, the result is a BINARY value. The default padding pattern in this case is the zero byte. When set to true, it restores the legacy behavior of always returning string types even for binary inputs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mssqlserver.datetimeoffsetMapping.enabled",
+      "doc": "When true, DATETIMEOFFSET is mapped to StringType; otherwise, it is mapped to TimestampType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mssqlserver.numericMapping.enabled",
+      "doc": "When true, use legacy MsSqlServer TINYINT, SMALLINT and REAL type mapping.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mysql.bitArrayMapping.enabled",
+      "doc": "When true, use LongType to represent MySQL BIT(n>1); otherwise, use BinaryType.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.mysql.timestampNTZMapping.enabled",
+      "doc": "When true, TimestampNTZType and MySQL TIMESTAMP can be converted bidirectionally. For reading, MySQL TIMESTAMP is converted to TimestampNTZType when JDBC read option preferTimestampNTZ is true. For writing, TimestampNTZType is converted to MySQL TIMESTAMP; otherwise, DATETIME",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.negativeIndexInArrayInsert",
+      "doc": "When set to true, restores the legacy behavior of `array_insert` for negative indexes - 0-based: the function inserts new element before the last one for the index -1. For example, `array_insert(['a', 'b'], -1, 'x')` returns `['a', 'x', 'b']`. When set to false, the -1 index points out to the last element, and the given example produces `['a', 'b', 'x']`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.noCharPaddingInPredicate",
+      "doc": "When true, Spark will not apply char type padding for CHAR type columns in string comparison predicates, when 'spark.sql.readSideCharPadding' is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.notReserveProperties",
+      "doc": "When true, all database and table properties are not reserved and available for create/alter syntaxes. But please be aware that the reserved properties will be silently removed.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.nullInEmptyListBehavior",
+      "doc": "When set to true, restores the legacy incorrect behavior of IN expressions for NULL values IN an empty list (including IN subqueries and literal IN lists): `null IN (empty list)` should evaluate to false, but sometimes (not always) incorrectly evaluates to null in the legacy behavior."
+    },
+    {
+      "key": "spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv",
+      "doc": "When set to false, nulls are written as unquoted empty strings in CSV data source. If set to true, it restores the legacy behavior that nulls were written as quoted empty strings, `\"\"`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.oracle.timestampMapping.enabled",
+      "doc": "When true, TimestampType maps to TIMESTAMP in Oracle; otherwise, TIMESTAMP WITH LOCAL TIME ZONE.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parameterSubstitution.constantsOnly",
+      "doc": "When true, limits parameter substitution to constants in DML/queries only, restoring the legacy behavior where parameter markers (? or :param) are only allowed in contexts where constant literals are expected. When false (default), parameter substitution is enabled everywhere a literal is supported, allowing parameter markers in any literal context throughout SQL parsing.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parquet.datetimeRebaseModeInWrite",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.parquet.datetimeRebaseModeInWrite' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.parquet.int96RebaseModeInRead",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.parquet.int96RebaseModeInRead' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.parquet.int96RebaseModeInWrite",
+      "doc": "(Removed)",
+      "defaultValue": "CORRECTED",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "Use 'spark.sql.parquet.int96RebaseModeInWrite' instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.parquet.nanosAsLong",
+      "doc": "When true, the Parquet's nanos precision timestamps are converted to SQL long values.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing",
+      "doc": "When true, if all requested fields of a struct are missing in a parquet file, assume the struct is always null, even if other fields are present. The default behavior is to fetch and read an arbitrary non-requested field present in the file to determine struct nullness. If enabled, schema pruning may cause non-null structs to be read as null.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parseNullPartitionSpecAsStringLiteral",
+      "doc": "If it is set to true, `PARTITION(col=null)` is parsed as a string literal of its text representation, e.g., string 'null', when the partition column is string type. Otherwise, it is always parsed as a null literal in the partition spec.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parseQueryWithoutEof",
+      "doc": "When set to true, ParserInterface#parseQuery(...) is going to use base `query` grammar term without EOF resulting in some queries (like `SELECT 1 UNION SELECT 2`) to be parsed incorrectly - `UNION` will be treated as an alias, and the rest of SQL input will be thrown away.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.parser.havingWithoutGroupByAsWhere",
+      "doc": "If it is set to true, the parser will treat HAVING without GROUP BY as a normal WHERE, which does not follow SQL standard.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.pathOptionBehavior.enabled",
+      "doc": "When true, \"path\" option is overwritten if one path parameter is passed to DataFrameReader.load(), DataFrameWriter.save(), DataStreamReader.load(), or DataStreamWriter.start(). Also, \"path\" option is added to the overall paths if multiple path parameters are passed to DataFrameReader.load()",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.percentileDiscCalculation",
+      "doc": "If true, the old bogus percentile_disc calculation is used. The old calculation incorrectly mapped the requested percentile to the sorted range of values in some cases and so returned incorrect results. Also, the new implementation is faster as it doesn't contain the interpolation logic that the old percentile_cont based one did.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.persistentCatalogFirst",
+      "doc": "When true (legacy), partially qualified function names like 'builtin.func' or 'session.func' resolve to the persistent catalog first. When false (default), the system catalog is prioritized (system.builtin.func or system.session.func).",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.postgres.datetimeMapping.enabled",
+      "doc": "When true, TimestampType maps to TIMESTAMP WITHOUT TIME ZONE in PostgreSQL for writing; otherwise, TIMESTAMP WITH TIME ZONE. When true, TIMESTAMP WITH TIME ZONE can be converted to TimestampNTZType when JDBC read option preferTimestampNTZ is true; otherwise, converted to TimestampType regardless of preferTimestampNTZ.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.raiseErrorWithoutErrorClass",
+      "doc": "When set to true, restores the legacy behavior of `raise_error` and `assert_true` to not return the `[USER_RAISED_EXCEPTION]` prefix. For example, `raise_error('error!')` returns `error!` instead of `[USER_RAISED_EXCEPTION] Error!`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.readFileSourceTableCacheIgnoreOptions",
+      "doc": "When set to true, reading from file source table caches the first query plan and ignores subsequent changes in query options. Otherwise, query options will be applied to the cached plan and may produce different results.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.replaceDatabricksSparkAvro.enabled",
+      "doc": "If it is set to true, the data source provider com.databricks.spark.avro is mapped to the built-in but external Avro data source module for backward compatibility.",
+      "defaultValue": "true",
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use `.format(\"avro\")` in `DataFrameWriter` or `DataFrameReader` instead."
+      }
+    },
+    {
+      "key": "spark.sql.legacy.respectNullabilityInTextDatasetConversion",
+      "doc": "When true, the nullability in the user-specified schema for `DataFrameReader.schema(schema).json(jsonDataset)` and `DataFrameReader.schema(schema).csv(csvDataset)` is respected. Otherwise, they are turned to a nullable schema forcibly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.scalarSubqueryAllowGroupByNonEqualityCorrelatedPredicate",
+      "doc": "When set to true, use incorrect legacy behavior for checking whether a scalar subquery with a group-by on correlated columns is allowed. See SPARK-48503",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.scalarSubqueryCountBugBehavior",
+      "doc": "When set to true, restores legacy behavior of potential incorrect count bug handling for scalar subqueries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.sessionInitWithConfigDefaults",
+      "doc": "Flag to revert to legacy behavior where a cloned SparkSession receives SparkConf defaults, dropping any overrides in its parent SparkSession.",
+      "defaultValue": "false",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.legacy.setCommandRejectsSparkCoreConfs",
+      "doc": "If it is set to true, SET command will fail when the key is registered as a SparkConf entry.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.setopsPrecedence.enabled",
+      "doc": "When set to true and the order of evaluation is not specified by parentheses, the set operations are performed from left to right as they appear in the query. When set to false and order of evaluation is not specified by parentheses, INTERSECT operations are performed before any UNION, EXCEPT and MINUS operations.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.sizeOfNull",
+      "doc": "If it is set to false, or spark.sql.ansi.enabled is true, then size of null returns null. Otherwise, it returns -1, which was inherited from Hive.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.skipTypeValidationOnAlterPartition",
+      "doc": "When true, skip validation for partition spec in ALTER PARTITION. E.g., `ALTER TABLE .. ADD PARTITION(p='a')` would work even the partition type is int. Besides, this config will also be used to skip type validation on partition spec when reading partitioned table. E.g., if the table partition spec is added without type validation, it might not be read correctly with the type validation. When false, the behavior follows spark.sql.storeAssignmentPolicy",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.statisticalAggregate",
+      "doc": "When set to true, statistical aggregate function returns Double.NaN if divide by zero occurred during expression evaluation, otherwise, it returns null. Before version 3.1.0, it returns NaN in divideByZero case by default.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.storeAnalyzedPlanForView",
+      "doc": "When true, analyzed plan instead of SQL text will be stored when creating temporary view",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.timeParserPolicy",
+      "doc": "When LEGACY, java.text.SimpleDateFormat is used for formatting and parsing dates/timestamps in a locale-sensitive manner, which is the approach before Spark 3.0. When set to CORRECTED, classes from java.time.* packages are used for the same purpose. When set to EXCEPTION, RuntimeException is thrown when we will get different results. The default is CORRECTED.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.legacy.truncateForEmptyRegexSplit",
+      "doc": "When set to true, splitting a string of length n using an empty regex with a positive limit discards the last n - limit characters.For example: SELECT split('abcd', '', 2) returns ['a', 'b'].When set to false, the last element of the resulting array contains all input beyond the last matched regex.For example: SELECT split('abcd', '', 2) returns ['a', 'bcd'].According to the description of the split function, this should be set to false by default. See SPARK-49968 for details.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.typeCoercion.datetimeToString.enabled",
+      "doc": "If it is set to true, date/timestamp will cast to string in binary comparisons with String when spark.sql.ansi.enabled is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useCurrentConfigsForView",
+      "doc": "When true, SQL Configs of the current active SparkSession instead of the captured ones will be applied during the parsing and analysis phases of the view resolution.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useLegacyXMLParser",
+      "doc": "When set to true, use the legacy XML parser for parsing XML files. Compared to the default parser, the legacy parser has less stringent validation checks for malformed content, but it's less memory-efficient",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.useV1Command",
+      "doc": "When true, Spark will use legacy V1 SQL commands.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.v1IdentifierNoCatalog",
+      "doc": "When set to false, the v1 identifier will include 'spark_catalog' as the catalog name if database is defined. When set to true, it restores the legacy behavior that does not include catalog name.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.legacy.viewSchemaBindingMode",
+      "doc": "Set to false to disable the WITH SCHEMA clause for view DDL and suppress the line in DESCRIBE EXTENDED and SHOW CREATE TABLE.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.legacy.viewSchemaCompensation",
+      "doc": "Set to false to revert default view schema binding mode from WITH SCHEMA COMPENSATION to WITH SCHEMA BINDING.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.lightweightPlanChangeValidation",
+      "doc": "Similar to spark.sql.planChangeValidation, this validates plan changes and runs after every rule, however it is enabled by default and so it should be lightweight.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.limit.initialNumPartitions",
+      "doc": "Initial number of partitions to try when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as more jobs will be run",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.limit.scaleUpFactor",
+      "doc": "Minimal increase rate in number of partitions between attempts when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as more jobs will be run",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.listagg.allowDistinctCastWithOrder.enabled",
+      "doc": "When true, LISTAGG(DISTINCT expr) WITHIN GROUP (ORDER BY expr) is allowed on non-string expr when the implicit cast to string preserves equality (e.g., integer, decimal, date). When false, the function argument and ORDER BY expression must have the exact same type, which requires explicit casts.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.mapKeyDedupPolicy",
+      "doc": "The policy to deduplicate map keys in builtin function: CreateMap, MapFromArrays, MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted at last takes precedence.",
+      "defaultValue": "EXCEPTION"
+    },
+    {
+      "key": "spark.sql.mapZipWithUsesJavaCollections",
+      "doc": "When true, the `map_zip_with` function uses Java collections instead of Scala collections. This is useful for avoiding NaN equality issues.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.maven.additionalRemoteRepositories",
+      "doc": "A comma-delimited string config of the optional additional remote Maven mirror repositories. This is only used for downloading Hive jars in IsolatedClientLoader if the default Maven Central repo is unreachable.",
+      "defaultValue": "https://maven-central.storage-download.googleapis.com/maven2/"
+    },
+    {
+      "key": "spark.sql.maxBroadcastTableSize",
+      "doc": "The maximum table size in bytes that can be broadcast in broadcast joins.",
+      "defaultValue": "8589934592b"
+    },
+    {
+      "key": "spark.sql.maxConcurrentOutputFileWriters",
+      "doc": "Maximum number of output file writers to use concurrently. If number of writers needed reaches this limit, task will sort rest of output then writing them.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.maxMetadataStringLength",
+      "doc": "Maximum number of characters to output for a metadata string. e.g. file location in `DataSourceScanExec`, every value will be abbreviated if exceed length.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.maxPlanStringLength",
+      "doc": "Maximum number of characters to output for a plan string.  If the plan is longer, further output will be truncated.  The default setting always generates a full plan.  Set this to a lower value such as 8k if plan strings are taking up too much memory or are causing OutOfMemory errors in the driver or UI processes.",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.maxSinglePartitionBytes",
+      "doc": "The maximum number of bytes allowed for a single partition. Otherwise, The planner will introduce shuffle to improve parallelism.",
+      "defaultValue": "128m"
+    },
+    {
+      "key": "spark.sql.mergeNestedTypeCoercion.enabled",
+      "doc": "If enabled, allow MERGE INTO to coerce source nested types if they have lessnested fields than the target table's nested types. This is experimental andthe semantics may change.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.metadataCacheTTLSeconds",
+      "doc": "Time-to-live (TTL) value for the metadata caches: partition file metadata cache and session catalog cache. This configuration only has an effect when this value having a positive value (> 0). It also requires setting 'spark.sql.catalogImplementation' to `hive`, setting 'spark.sql.hive.filesourcePartitionFileCacheSize' > 0 and setting 'spark.sql.hive.manageFilesourcePartitions' to `true` to be applied to the partition file metadata cache.",
+      "defaultValue": "-1000ms",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.nameResolutionLog.level",
+      "doc": "Configures the log level for logging the name resolution in the single-pass bottom-up Resolver. The value can be ERROR, WARN, INFO, DEBUG, TRACE.",
+      "defaultValue": "TRACE"
+    },
+    {
+      "key": "spark.sql.objectHashAggregate.sortBased.fallbackThreshold",
+      "doc": "In the case of ObjectHashAggregateExec, when the size of the in-memory hash map grows too large, we will fall back to sort-based aggregation. This option sets a row count threshold for the size of the hash map.",
+      "defaultValue": "128"
+    },
+    {
+      "key": "spark.sql.operatorPipeSyntaxEnabled",
+      "doc": "If true, enable operator pipe syntax for Apache Spark SQL. This uses the operator pipe marker |> to indicate separation between clauses of SQL in a manner that describes the sequence of steps that the query performs in a composable fashion.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizeNullAwareAntiJoin",
+      "doc": "When true, NULL-aware anti join execution will be planed into BroadcastHashJoinExec with flag isNullAwareAntiJoin enabled, optimized from O(M*N) calculation into O(M) calculation using Hash lookup instead of Looping lookup. Only support for singleColumn NAAJ for now.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.avoidCollapseUDFWithExpensiveExpr",
+      "doc": "Whether to avoid collapsing projections that would duplicate expensive expressions in UDFs.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.avoidDoubleFilterEval",
+      "doc": "When true avoid pushing expensive (UDF, etc.) filters down if it could result indouble evaluation. This was the behaviour prior to 3.X.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.canChangeCachedPlanOutputPartitioning",
+      "doc": "Whether to forcibly enable some optimization rules that can change the output partitioning of a cached query when executing it for caching. If it is set to true, queries may need an extra shuffle to read the cached data. This configuration is disabled by default. The optimization rule enabled by this configuration is spark.sql.adaptive.applyFinalStageShuffleOptimizations.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.collapseProjectAlwaysInline",
+      "doc": "Whether to always collapse two adjacent projections and inline expressions even if it causes extra duplication.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.datasourceV2ExprFolding",
+      "doc": "When this config is set to true, do safe constant folding for the expressions before translation and pushdown.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.datasourceV2JoinPushdown",
+      "doc": "When this config is set to true, join is tried to be pushed downfor DSv2 data sources in V2ScanRelationPushdown optimization rule.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateExistsSubqueryLegacyIncorrectCountHandling.enabled",
+      "doc": "If enabled, revert to legacy incorrect behavior for certain EXISTS/IN subqueries with COUNT or similar aggregates.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateInnerQuery.enabled",
+      "doc": "Decorrelate inner query by eliminating correlated references and build domain joins.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateJoinPredicate.enabled",
+      "doc": "Decorrelate scalar and lateral subqueries with correlated references in join predicates. This configuration is only effective when 'spark.sql.optimizer.decorrelateInnerQuery.enabled' is true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateLimit.enabled",
+      "doc": "Decorrelate subqueries with correlation under LIMIT.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateOffset.enabled",
+      "doc": "Decorrelate subqueries with correlation under LIMIT with OFFSET.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelatePredicateSubqueriesInJoinPredicate.enabled",
+      "doc": "Decorrelate predicate (in and exists) subqueries with correlated references in join predicates.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSetOps.enabled",
+      "doc": "Decorrelate subqueries with correlation under set operators.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSubqueryLegacyIncorrectCountHandling.enabled",
+      "doc": "If enabled, revert to legacy incorrect behavior for certain subqueries with COUNT or similar aggregates: see SPARK-43098.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateSubqueryPreventConstantHoldingForCountBug.enabled",
+      "doc": "If enabled, prevents constant folding in subqueries that contain a COUNT-bug-susceptible Aggregate.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.decorrelateUnionOrSetOpUnderLimit.enabled",
+      "doc": "Decorrelate UNION or SET operation under LIMIT operator. If not enabled,revert to legacy incorrect behavior for certain subqueries with correlation underUNION/SET operator with a LIMIT operator above it.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.disableHints",
+      "doc": "When true, the optimizer will disable user-specified hints that are additional directives for better planning of a query.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.enabled",
+      "doc": "When true, we will generate predicate for partition column when it's used as join key",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio",
+      "doc": "When statistics are not available or configured not to be used, this config will be used as the fallback filter ratio for computing the data size of the partitioned table after dynamic partition pruning, in order to evaluate if it is worth adding an extra subquery as the pruning filter if broadcast reuse is not applicable.",
+      "defaultValue": "0.5"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly",
+      "doc": "When true, dynamic partition pruning will only apply when the broadcast exchange of a broadcast hash join operation can be reused as the dynamic pruning filter.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.dynamicPartitionPruning.useStats",
+      "doc": "When true, distinct count statistics will be used for computing the data size of the partitioned table after dynamic partition pruning, in order to evaluate if it is worth adding an extra subquery as the pruning filter if broadcast reuse is not applicable.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.enableCsvExpressionOptimization",
+      "doc": "Whether to optimize CSV expressions in SQL optimizer. It includes pruning unnecessary columns from from_csv.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.enableJsonExpressionOptimization",
+      "doc": "Whether to optimize JSON expressions in SQL optimizer. It includes pruning unnecessary columns from from_json, simplifying from_json + to_json, to_json + named_struct(from_json.col1, from_json.col2, ....).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.excludeSubqueryRefsFromRemoveRedundantAliases.enabled",
+      "doc": "When true, exclude the references from the subquery expressions (in, exists, etc.) while removing redundant aliases.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.excludedRules",
+      "doc": "Configures a list of rules to be disabled in the optimizer, in which the rules are specified by their rule names and separated by comma. It is not guaranteed that all the rules in this configuration will eventually be excluded, as some rules are necessary for correctness. The optimizer will log the rules that have indeed been excluded."
+    },
+    {
+      "key": "spark.sql.optimizer.expression.nestedPruning.enabled",
+      "doc": "Prune nested fields from expressions in an operator which are unnecessary in satisfying a query. Note that this optimization doesn't prune nested fields from physical data source scanning. For pruning nested fields from scanning, please use `spark.sql.optimizer.nestedSchemaPruning.enabled` config.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.expressionProjectionCandidateLimit",
+      "doc": "The maximum number of the candidate of output expressions whose alias are replaced. It can preserve the output partitioning and ordering. Negative value means disable this optimization.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.optimizer.inSetConversionThreshold",
+      "doc": "The threshold of set size for InSet conversion.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.optimizer.inSetSwitchThreshold",
+      "doc": "Configures the max set size in InSet for which Spark will generate code with switch statements. This is applicable only to bytes, shorts, ints, dates.",
+      "defaultValue": "400"
+    },
+    {
+      "key": "spark.sql.optimizer.mapLookupHashThreshold",
+      "doc": "The minimum number of map entries to attempt hash-based lookup in `element_at` and the `[]` operator. Only applies to foldable map expressions (constants / literals), where the hash index can be built once and reused across all rows. Non-foldable maps always use linear scan to avoid per-row hash-table rebuild overhead. Below this threshold, linear scan is used. For key types that do not support hashing (e.g. arrays, structs, binary), linear scan is always used regardless of map size.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.optimizer.maxIterations",
+      "doc": "The max number of iterations the optimizer runs.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.optimizer.mergeSubplans.filterPropagation.enabled",
+      "doc": "When set to true, subquery plans that differ only in their filter conditions can be merged by propagating filters up to enclosing non-grouping aggregates.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.mergeSubplans.filterPropagation.symmetricFilterPropagation.enabled",
+      "doc": "When set to true, two non-grouping aggregate subplans that both have filter conditions (but with different predicates) can be merged into a single scan using FILTER (WHERE ...) clauses on each aggregate expression. Merging two filtered scans broadens the combined filter to OR(f1, f2), which may reduce IO pruning (e.g. partition or file skipping) compared to the individual filters. Disabled by default; enable once the behaviour has been validated in your workload, particularly on heavily partitioned or file-pruned tables. Has no effect when spark.sql.optimizer.mergeSubplans.filterPropagation.enabled is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.mergeSubplans.filterPropagation.throughJoin.enabled",
+      "doc": "When set to true, filter attributes can propagate through Join nodes during subplan merging, allowing subplans that differ only in their filter conditions and share a common join to be merged into a single scan. A filter attribute is only propagated through a join when it originates from the non-nullable (preserved) side: the left side of LeftOuter/LeftSemi/LeftAnti, the right side of RightOuter, or either side of Inner/Cross. FullOuter joins are never eligible. Has no effect when spark.sql.optimizer.mergeSubplans.filterPropagation.enabled is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.metadataOnly",
+      "doc": "When true, enable the metadata-only query optimization that use the table's metadata to produce the partition columns instead of table scans. It applies when all the columns scanned are partition columns and the query has an aggregate operator that satisfies distinct semantics. By default the optimization is disabled, and deprecated as of Spark 3.0 since it may return incorrect results when the files are empty, see also SPARK-26709. It will be removed in the future releases. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "3.0",
+        "comment": "Avoid to depend on this optimization to prevent a potential correctness issue. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources",
+      "doc": "A comma-separated list of data source short names or fully qualified data source implementation class names for which Spark tries to push down predicates for nested columns and/or names containing `dots` to data sources. This configuration is only effective with file-based data sources in DSv1. Currently, Parquet and ORC implement both optimizations. The other data sources don't support this feature yet. So the default value is 'parquet,orc'.",
+      "defaultValue": "parquet,orc"
+    },
+    {
+      "key": "spark.sql.optimizer.nestedSchemaPruning.enabled",
+      "doc": "Prune nested fields from a logical relation's output which are unnecessary in satisfying a query. This optimization allows columnar file format readers to avoid reading unnecessary nested column data. Currently Parquet and ORC are the data sources that implement this optimization.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeExpandRatio",
+      "doc": "Threshold for inserting a pre-aggregation step before the Expand operator produced by RewriteDistinctAggregates. Queries with multiple COUNT(DISTINCT) on different columns are rewritten using an Expand that duplicates each row N times (one per distinct group). When N >= this threshold, a de-duplication aggregate on (grouping keys + all distinct columns) is inserted before the Expand to reduce the amplified data volume. Only applies to pure distinct aggregates without non-distinct aggregates or FILTER clauses. Set to -1 to disable.",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery",
+      "doc": "When true, the optimizer will inline subqueries with OneRowRelation as leaf nodes.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline",
+      "doc": "When true, the optimizer will always inline single row subqueries even if it causes extra duplication. It only takes effect when spark.sql.optimizer.optimizeOneRowRelationSubquery is set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.optimizeUncorrelatedInSubqueriesInJoinCondition.enabled",
+      "doc": "When true, optimize uncorrelated IN subqueries in join predicates by rewriting them to joins. This interacts with spark.sql.legacy.nullInEmptyListBehavior because it can rewrite IN predicates.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.batches",
+      "doc": "(Removed)",
+      "defaultValue": "",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.batches` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.level",
+      "doc": "(Removed)",
+      "defaultValue": "trace",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.level` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.planChangeLog.rules",
+      "doc": "(Removed)",
+      "defaultValue": "",
+      "removed": {
+        "version": "3.1.0",
+        "comment": "Please use `spark.sql.planChangeLog.rules` instead."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.plannedWrite.enabled",
+      "doc": "When set to true, Spark optimizer will add logical sort operators to V1 write commands if needed so that `FileFormatWriter` does not need to insert physical sorts.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.preserveAliasMetadataWhenCollapsingProjects",
+      "doc": "When true, make sure to explicitly copy the metadata of the aliases from lower project list.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.propagateDistinctKeys.enabled",
+      "doc": "When true, the query optimizer will propagate a set of distinct attributes from the current node and use it to optimize query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.pruneFiltersCanPruneStreamingSubplan",
+      "doc": "Allow PruneFilters to remove streaming subplans when we encounter a false filter. This flag is to restore prior buggy behavior for broken pipelines.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.pullHintsIntoSubqueries",
+      "doc": "Pull hints into subqueries in EliminateResolvedHint if enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.pullOutNestedDataOuterRefExpressions.enabled",
+      "doc": "Handle correlation over nested data extract expressions by pulling out the expression into the outer plan. This enables correlation on map attributes for example.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.pushDownJoinThroughUnion.enabled",
+      "doc": "When true, pushes down Join through Union when the right side is small enough to broadcast. This can improve performance by allowing each Union branch to directly perform a broadcast join, avoiding materializing the entire Union result.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.optimizer.replaceExceptWithFilter",
+      "doc": "When true, the apply function of the rule verifies whether the right node of the except operation is of type Filter or Project followed by Filter. If yes, the rule further verifies 1) Excluding the filter operations from the right (as well as the left node, if any) on the top, whether both the nodes evaluates to a same result. 2) The left and right nodes don't contain any SubqueryExpressions. 3) The output column names of the left node are distinct. If all the conditions are met, the rule will replace the except operation with a Filter by flipping the filter condition(s) of the right node.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold",
+      "doc": "Byte size threshold of the Bloom filter application side plan's aggregated scan size. Aggregated scan byte size of the Bloom filter application side needs to be over this value to inject a bloom filter.",
+      "defaultValue": "10GB"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold",
+      "doc": "Size threshold of the bloom filter creation side plan. Estimated size needs to be under this value to try to inject bloom filter.",
+      "defaultValue": "10MB"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.enabled",
+      "doc": "When true and if one side of a shuffle join has a selective predicate, we attempt to insert a bloom filter in the other side to reduce the amount of shuffle data.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.expectedNumItems",
+      "doc": "The default number of expected items for the runtime bloomfilter",
+      "defaultValue": "1000000"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.maxNumBits",
+      "doc": "The max number of bits to use for the runtime bloom filter",
+      "defaultValue": "67108864"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.maxNumItems",
+      "doc": "The max allowed number of expected items for the runtime bloom filter",
+      "defaultValue": "4000000"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.bloomFilter.numBits",
+      "doc": "The default number of bits to use for the runtime bloom filter",
+      "defaultValue": "8388608"
+    },
+    {
+      "key": "spark.sql.optimizer.runtime.rowLevelOperationGroupFilter.enabled",
+      "doc": "Enables runtime filtering for group-based and delta-based row-level operations. Data sources may prune entire file groups at runtime when planning a row-level operation scan. Planning-time filter pushdown is limited as not all expressions can be converted into data source filters and some expressions can only be evaluated by Spark (e.g. subqueries). Since rewriting groups or scanning unnecessary files is expensive, Spark can execute a lightweight query at runtime to find what records match the condition of the row-level operation. The information about matching records will be passed back to the row-level operation scan, allowing data sources to skip files that don't have to be processed.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.runtimeFilter.number.threshold",
+      "doc": "The total number of injected runtime filters (non-DPP) for a single query. This is to prevent driver OOMs with too many Bloom filters.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.optimizer.runtimeFilter.semiJoinReduction.enabled",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "4.0.0",
+        "comment": "This optimizer config is useless as runtime filter cannot be an IN subquery now."
+      }
+    },
+    {
+      "key": "spark.sql.optimizer.scalarSubqueryUseSingleJoin",
+      "doc": "When set to true, use LEFT_SINGLE join for correlated scalar subqueries where optimizer can't prove that only 1 row will be returned",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.serializer.nestedSchemaPruning.enabled",
+      "doc": "Prune nested fields from object serialization operator which are unnecessary in satisfying a query. This optimization allows object serializers to avoid executing unnecessary nested expressions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.optimizer.windowGroupLimitThreshold",
+      "doc": "Threshold for triggering `InsertWindowGroupLimit`. 0 means the output results is empty. -1 means disabling the optimization.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.optimizer.wrapExistsInAggregateFunction",
+      "doc": "When true, the optimizer will wrap newly introduced `exists` attributes in an aggregate function to ensure that Aggregate nodes preserve semantic invariant that each variable among agg expressions appears either in grouping expressions or belongs to and aggregate function.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.aggregatePushdown",
+      "doc": "If true, aggregates will be pushed down to ORC for optimization. Support MIN, MAX and COUNT as aggregate expression. For MIN/MAX, support boolean, integer, float and date type. For COUNT, support all data types. If statistics is missing from any ORC file footer, exception would be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.orc.columnarReaderBatchSize",
+      "doc": "The number of rows to include in a orc vectorized reader batch. The number should be carefully chosen to minimize overhead and avoid OOMs in reading data.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.orc.columnarWriterBatchSize",
+      "doc": "The number of rows to include in a orc vectorized writer batch. The number should be carefully chosen to minimize overhead and avoid OOMs in writing data.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.orc.compression.codec",
+      "doc": "Sets the compression codec used when writing ORC files. If either `compression` or `orc.compress` is specified in the table-specific options/properties, the precedence would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`. Acceptable values include: none, uncompressed, snappy, zlib, lzo, zstd, lz4, brotli.",
+      "defaultValue": "zstd"
+    },
+    {
+      "key": "spark.sql.orc.enableNestedColumnVectorizedReader",
+      "doc": "Enables vectorized orc decoding for nested column.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.enableVectorizedReader",
+      "doc": "Enables vectorized orc decoding.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.filterPushdown",
+      "doc": "When true, enable filter pushdown for ORC files.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orc.impl",
+      "doc": "When native, use the native version of ORC support instead of the ORC library in Hive. It is 'hive' by default prior to Spark 2.4.",
+      "defaultValue": "native"
+    },
+    {
+      "key": "spark.sql.orc.mergeSchema",
+      "doc": "When true, the Orc data source merges schemas collected from all data files, otherwise the schema is picked from a random data file.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.orderByOrdinal",
+      "doc": "When true, the ordinal numbers are treated as the position in the select list. When false, the ordinal numbers in order/sort by clause are ignored.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.orderingAwareLimitOffset",
+      "doc": "When set to true, a local sort will be inserted between GlobalLimitExec and single-partition ShuffleExchangeExec, if the underlying plan produces sorted data. This is because shuffle reader in Spark fetches shuffle blocks in a random order and can not preserve the data ordering, while LIMIT/OFFSET must preserve ordering.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.aggregatePushdown",
+      "doc": "If true, aggregates will be pushed down to Parquet for optimization. Support MIN, MAX and COUNT as aggregate expression. For MIN/MAX, support boolean, integer, float and date type. For COUNT, support all data types. If statistics is missing from any Parquet file footer, exception would be thrown.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.binaryAsString",
+      "doc": "Some other Parquet-producing systems, in particular Impala and older versions of Spark SQL, do not differentiate between binary data and strings when writing out the Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.columnarReaderBatchSize",
+      "doc": "The number of rows to include in a parquet vectorized reader batch. The number should be carefully chosen to minimize overhead and avoid OOMs in reading data.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.parquet.compression.codec",
+      "doc": "Sets the compression codec used when writing Parquet files. If either `compression` or `parquet.compression` is specified in the table-specific options/properties, the precedence would be `compression`, `parquet.compression`, `spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd.",
+      "defaultValue": "snappy"
+    },
+    {
+      "key": "spark.sql.parquet.datetimeRebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. When EXCEPTION, Spark will fail the reading if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown. This config influences on reads of the following parquet logical types: DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: spark.sql.parquet.int96RebaseModeInRead.",
+      "defaultValue": "CORRECTED",
+      "alternatives": [
+        "spark.sql.legacy.parquet.datetimeRebaseModeInRead"
+      ]
+    },
+    {
+      "key": "spark.sql.parquet.datetimeRebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. When EXCEPTION, Spark will fail the writing if it sees ancient dates/timestamps that are ambiguous between the two calendars. This config influences on writes of the following parquet logical types: DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS. The INT96 type has the separate config: spark.sql.parquet.int96RebaseModeInWrite.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.parquet.enableNestedColumnVectorizedReader",
+      "doc": "Enables vectorized Parquet decoding for nested columns (e.g., struct, list, map). Requires spark.sql.parquet.enableVectorizedReader to be enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.enableNullTypeVectorizedReader",
+      "doc": "Enables vectorized Parquet reader support for NullType columns.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.enableVectorizedReader",
+      "doc": "Enables vectorized parquet decoding.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.read.enabled",
+      "doc": "Field ID is a native field of the Parquet schema spec. When enabled, Parquet readers will use field IDs (if present) in the requested Spark schema to look up Parquet fields instead of using column names",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.read.ignoreMissing",
+      "doc": "When the Parquet file doesn't have any field IDs but the Spark read schema is using field IDs to read, we will silently return nulls when this flag is enabled, or error otherwise.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.fieldId.write.enabled",
+      "doc": "Field ID is a native field of the Parquet schema spec. When enabled, Parquet writers will populate the field Id metadata (if present) in the Spark schema to the Parquet schema.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown",
+      "doc": "Enables Parquet filter push-down optimization when set to true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.date",
+      "doc": "If true, enables Parquet filter push-down optimization for Date. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.decimal",
+      "doc": "If true, enables Parquet filter push-down optimization for Decimal. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.string.startsWith",
+      "doc": "If true, enables Parquet filter push-down optimization for string startsWith function. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.stringPredicate",
+      "doc": "If true, enables Parquet filter push-down optimization for string predicate such as startsWith/endsWith/contains function. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "fallback": "spark.sql.parquet.filterPushdown.string.startsWith"
+    },
+    {
+      "key": "spark.sql.parquet.filterPushdown.timestamp",
+      "doc": "If true, enables Parquet filter push-down optimization for Timestamp. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.ignoreVariantAnnotation",
+      "doc": "When true, ignore the variant logical type annotation and treat the Parquet column in the same way as the underlying struct type",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.inferTimestampNTZ.enabled",
+      "doc": "When enabled, Parquet timestamp columns with annotation isAdjustedToUTC = false are inferred as TIMESTAMP_NTZ type during schema inference. Otherwise, all the Parquet timestamp columns are inferred as TIMESTAMP_LTZ types. Note that Spark writes the output schema into Parquet's footer metadata on file writing and leverages it on file reading. Thus this configuration only affects the schema inference on Parquet files which are not written by Spark.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.int64AsTimestampMillis",
+      "doc": "(Removed)",
+      "defaultValue": "false",
+      "removed": {
+        "version": "3.0.0",
+        "comment": "The config was deprecated since Spark 2.3. Use 'spark.sql.parquet.outputTimestampType' instead of it."
+      }
+    },
+    {
+      "key": "spark.sql.parquet.int96AsTimestamp",
+      "doc": "Some Parquet-producing systems, in particular Impala, store Timestamp into INT96. Spark would also store Timestamp as INT96 because we need to avoid precision lost of the nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.int96RebaseModeInRead",
+      "doc": "When LEGACY, Spark will rebase INT96 timestamps from the legacy hybrid (Julian + Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. When CORRECTED, Spark will not do rebase and read the timestamps as it is. When EXCEPTION, Spark will fail the reading if it sees ancient timestamps that are ambiguous between the two calendars. This config is only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.parquet.int96RebaseModeInWrite",
+      "doc": "When LEGACY, Spark will rebase INT96 timestamps from Proleptic Gregorian calendar to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. When CORRECTED, Spark will not do rebase and write the timestamps as it is. When EXCEPTION, Spark will fail the writing if it sees ancient timestamps that are ambiguous between the two calendars.",
+      "defaultValue": "CORRECTED"
+    },
+    {
+      "key": "spark.sql.parquet.int96TimestampConversion",
+      "doc": "This controls whether timestamp adjustments should be applied to INT96 data when converting to timestamps, for data written by Impala.  This is necessary because Impala stores INT96 data with a different timezone offset than Hive & Spark.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.mergeSchema",
+      "doc": "When true, the Parquet data source merges schemas collected from all data files, otherwise the schema is picked from the summary file or a random data file if no summary file is available.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.output.committer.class",
+      "doc": "The output committer class used by Parquet. The specified class needs to be a subclass of org.apache.hadoop.mapreduce.OutputCommitter. Typically, it's also a subclass of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata summaries will never be created, irrespective of the value of parquet.summary.metadata.level",
+      "defaultValue": "org.apache.parquet.hadoop.ParquetOutputCommitter"
+    },
+    {
+      "key": "spark.sql.parquet.outputTimestampType",
+      "doc": "Sets which Parquet timestamp type to use when Spark writes data to Parquet files. INT96 is a non-standard but commonly used timestamp type in Parquet. TIMESTAMP_MICROS is a standard timestamp type in Parquet, which stores number of microseconds from the Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which means Spark has to truncate the microsecond portion of its timestamp value.",
+      "defaultValue": "INT96"
+    },
+    {
+      "key": "spark.sql.parquet.pushdown.inFilterThreshold",
+      "doc": "For IN predicate, Parquet filter will push-down a set of OR clauses if its number of values not exceeds this threshold. Otherwise, Parquet filter will push-down a value greater than or equal to its minimum value and less than or equal to its maximum value. By setting this value to 0 this feature can be disabled. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.parquet.reader.respectUnknownTypeAnnotation.enabled",
+      "doc": "When enabled, respects the UNKNOWN type annotation in Parquet files during schema inference and infers NullType. When disabled, ignores the UNKNOWN annotation and uses the physical type instead.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.recordLevelFilter.enabled",
+      "doc": "If true, enables Parquet's native record-level filtering using the pushed down filters. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and the vectorized reader is not used. You can ensure the vectorized reader is not used by setting 'spark.sql.parquet.enableVectorizedReader' to false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.respectSummaryFiles",
+      "doc": "When true, we make assumption that all part-files of Parquet are consistent with summary files and we will ignore them when merging schema. Otherwise, if this is false, which is the default, we will merge all part-files. This should be considered as expert-only option, and shouldn't be enabled before knowing what it means exactly.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parquet.variant.annotateLogicalType.enabled",
+      "doc": "When enabled, Spark annotates the variant groups written to Parquet as the parquet variant logical type.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parquet.writeLegacyFormat",
+      "doc": "If true, data will be written in a way of Spark 1.4 and earlier. For example, decimal values will be written in Apache Parquet's fixed-length byte array format, which other systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet will be used. For example, decimals will be written in int-based format. If Parquet output is intended for use with systems that do not support this newer format, set to true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parser.eagerEvalOfUnresolvedInlineTable",
+      "doc": "Controls whether we optimize the ASTree that gets generated when parsing VALUES lists (UnresolvedInlineTable) by eagerly evaluating it in the AST Builder.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.parser.escapedStringLiterals",
+      "doc": "When true, string literals (including regex patterns) remain escaped in our SQL parser. The default is false since Spark 2.0. Setting it to true can restore the behavior prior to Spark 2.0.",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "4.0",
+        "comment": "Use raw string literals with the `r` prefix instead. "
+      }
+    },
+    {
+      "key": "spark.sql.parser.manageParserCaches",
+      "doc": "When true, we install our own ANTLR caches to manage memory usage. When false, we use the\ndefault ANTLR caches. Dependency for\n`spark.sql.parser.parserDfaCacheFlushThreshold`.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parser.parserDfaCacheFlushRatio",
+      "doc": "Like `spark.sql.parser.parserDfaCacheFlushThreshold`, but uses a threshold that is a\nlinear function of the memory allocated to the driver process. Represents the percentage\nof the driver memory that the DFA cache can consume before it is flushed.\n\nEstimates the memory used by the DFA cache, assuming each state consumes\n`AbstractParser.BYTES_PER_DFA_STATE` bytes. If this value exceeds the product of the\ndriver memory with the config value (interpreted as a percentage), the cache is flushed.\n\nActive values should be in the range 0-100, and a negative value disables the feature.\nIf both this config and `spark.sql.parser.parserDfaCacheFlushThreshold` are set, the\ncache is flushed if either condition is met.\nRequires `spark.sql.parser.manageParserCaches` to be true to take effect.\n",
+      "defaultValue": "-1.0"
+    },
+    {
+      "key": "spark.sql.parser.parserDfaCacheFlushThreshold",
+      "doc": "When positive, release ANTLR caches after parsing a SQL query when the number of states\nin the DFA cache exceeds the value of the config. DFA states empirically consume about\n`AbstractParser.BYTES_PER_DFA_STATE` bytes of memory each.\n\nANTLR parsers retain a DFA cache designed to speed up parsing future input. However,\nthere is no limit to how large this cache can become. Parsing large SQL statements can\nlead to an accumulation of objects in the cache that are unlikely to be reused, causing\nhigh GC overhead and eventually OOMs.\n\nIf this config is set to a negative value, it is ignored.\nIf both this config and `spark.sql.parser.parserDfaCacheFlushRatio` are set, the\ncache is flushed if either condition is met.\nRequires `spark.sql.parser.manageParserCaches` to be true to take effect.\n\nCan significantly slow down parsing in exchange for better memory stability.\n",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.parser.quotedRegexColumnNames",
+      "doc": "When true, quoted Identifiers (using backticks) in SELECT statement are interpreted as regular expressions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.parser.singleCharacterPipeOperator.enabled",
+      "doc": "When true, the single character pipe token '|' can be used as an alternative to '|>' for SQL pipe operators. When false, only '|>' is recognized as a pipe operator, and '|' is only used for bitwise OR operations. This provides syntax compatibility with other languages like Splunk SPL and Kusto that use '|' for pipe operations.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.path.enabled",
+      "doc": "When true, enables the SQL Standard PATH feature: SET PATH, path-based routine resolution, and CURRENT_PATH(). When false, SET PATH is rejected and resolution uses the default path only.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pipelines.event.queue.capacity",
+      "doc": "Capacity of the event queue used in pipelined execution. When the queue is full, non-terminal FlowProgressEvents will be dropped.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.pipelines.execution.maxConcurrentFlows",
+      "doc": "Max number of flows to execute at once. Used to tune performance for triggered pipelines. Has no effect on continuous pipelines.",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.pipelines.execution.streamstate.pollingInterval",
+      "doc": "Interval in seconds at which the stream state is polled for changes. This is used to check if the stream has failed and needs to be restarted.",
+      "defaultValue": "1000ms"
+    },
+    {
+      "key": "spark.sql.pipelines.execution.watchdog.maxRetryTime",
+      "doc": "Maximum time interval in seconds at which flows will be restarted.",
+      "defaultValue": "3600000ms"
+    },
+    {
+      "key": "spark.sql.pipelines.execution.watchdog.minRetryTime",
+      "doc": "Initial duration in seconds between the time when we notice a flow has failed and when we try to restart the flow. The interval between flow restarts doubles with every stream failure up to the maximum value set in `pipelines.execution.watchdog.maxRetryTime`.",
+      "defaultValue": "5000ms"
+    },
+    {
+      "key": "spark.sql.pipelines.maxFlowRetryAttempts",
+      "doc": "Maximum number of times a flow can be retried",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.pipelines.timeoutMsForTerminationJoinAndLock",
+      "doc": "Timeout in milliseconds to grab a lock for stopping update - default is 1hr.",
+      "defaultValue": "3600000ms"
+    },
+    {
+      "key": "spark.sql.pivotMaxValues",
+      "doc": "When doing a pivot without specifying values for the pivot column this is the maximum number of (distinct) values that will be collected without error.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.planChangeLog.batches",
+      "doc": "Configures a list of batches for logging plan changes, in which the batches are specified by their batch names and separated by comma."
+    },
+    {
+      "key": "spark.sql.planChangeLog.level",
+      "doc": "Configures the log level for logging the change from the original plan to the new plan after a rule or batch is applied. The value can be ERROR, WARN, INFO, DEBUG, TRACE.",
+      "defaultValue": "TRACE"
+    },
+    {
+      "key": "spark.sql.planChangeLog.rules",
+      "doc": "Configures a list of rules for logging plan changes, in which the rules are specified by their rule names and separated by comma."
+    },
+    {
+      "key": "spark.sql.planChangeValidation",
+      "doc": "If true, Spark will validate all the plan changes made by analyzer/optimizer and other catalyst rules, to make sure every rule returns a valid plan",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.planner.pythonExecution.memory",
+      "doc": "Specifies the memory allocation for executing Python code in Spark driver, in MiB. When set, it caps the memory for Python execution to the specified amount. If not set, Spark will not limit Python's memory usage and it is up to the application to avoid exceeding the overhead memory space shared with other non-JVM processes.\nNote: Windows does not support resource limiting and actual resource is not limited on MacOS."
+    },
+    {
+      "key": "spark.sql.preserveCharVarcharTypeInfo",
+      "doc": "When true, Spark does not replace CHAR/VARCHAR types the STRING type, which is the default behavior of Spark 3.0 and earlier versions. This means the length checks for CHAR/VARCHAR types is enforced and CHAR type is also properly padded.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.prioritizeOrdinalResolutionInSort.enabled",
+      "doc": "When set to true, we prioritize ordinal resolution in Sort over other expressions. Otherwise, no order is enforced.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.pyspark.dataSource.profiler",
+      "doc": "Configure the Python Data Source profiler by enabling or disabling it with the option to choose between \"perf\" and \"memory\" types, or unsetting the config disables the profiler. This is disabled by default."
+    },
+    {
+      "key": "spark.sql.pyspark.inferNestedDictAsStruct.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the nested dict as a map by default. When it set to true, it infers the nested dict as a struct.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.jvmStacktrace.enabled",
+      "doc": "When true, it shows the JVM stacktrace in the user-facing PySpark exception together with Python stacktrace. By default, it is disabled to hide JVM stacktrace and shows a Python-friendly exception only. Note that this is independent from log level settings.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the element type of an array from all values in the array by default. If this config is set to true, it restores the legacy behavior of only inferring the type from the first array element.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.legacy.inferMapTypeFromFirstPair.enabled",
+      "doc": "PySpark's SparkSession.createDataFrame infers the key/value types of a map from all pairs in the map by default. If this config is set to true, it restores the legacy behavior of only inferring the type from the first non-null pair.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.plotting.max_rows",
+      "doc": "The visual limit on plots. If set to 1000 for top-n-based plots (pie, bar, barh), the first 1000 data points will be used for plotting. For sampled-based plots (scatter, area, line), 1000 data points will be randomly sampled.",
+      "defaultValue": "1000"
+    },
+    {
+      "key": "spark.sql.pyspark.toJSON.returnDataFrame",
+      "doc": "When true, DataFrame.toJSON in PySpark Classic returns a Dataframe instead of RDD.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.pyspark.udf.profiler",
+      "doc": "Configure the Python/Pandas UDF profiler by enabling or disabling it with the option to choose between \"perf\" and \"memory\" types, or unsetting the config disables the profiler. This is disabled by default."
+    },
+    {
+      "key": "spark.sql.pyspark.worker.logging.enabled",
+      "doc": "When set to true, this configuration enables comprehensive logging within Python worker processes that execute User-Defined Functions (UDFs), User-Defined Table Functions (UDTFs), and other Python-based operations in Spark SQL.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.python.filterPushdown.enabled",
+      "doc": "When true, enable filter pushdown to Python datasource, at the cost of running Python worker one additional time during planning.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.queryExecutionListeners",
+      "doc": "List of class names implementing QueryExecutionListener that will be automatically added to newly created sessions. The classes should have either a no-arg constructor, or a constructor that expects a SparkConf argument.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.readSideCharPadding",
+      "doc": "When true, Spark applies string padding when reading CHAR type columns/fields, in addition to the write-side padding. This config is true by default to better enforce CHAR type semantic in cases such as external tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.redaction.options.regex",
+      "doc": "Regex to decide which keys in a Spark SQL command's options map contain sensitive information. The values of options whose names that match this regex will be redacted in the explain output. This redaction is applied on top of the global redaction configuration defined by spark.redaction.regex.",
+      "defaultValue": "(?i)url"
+    },
+    {
+      "key": "spark.sql.redaction.string.regex",
+      "doc": "Regex to decide which parts of strings produced by Spark contain sensitive information. When this regex matches a string part, that string part is replaced by a dummy value. This is currently used to redact the output of SQL explain commands. When this conf is not set, the value from `spark.redaction.string.regex` is used.",
+      "fallback": "spark.redaction.string.regex"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.enabled",
+      "doc": "Enables eager evaluation or not. When true, the top K rows of Dataset will be displayed if and only if the REPL supports the eager evaluation. Currently, the eager evaluation is supported in PySpark and SparkR. In PySpark, for the notebooks like Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned outputs are showed similar to R data.frame would.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.maxNumRows",
+      "doc": "The max number of rows that are returned by eager evaluation. This only takes effect when spark.sql.repl.eagerEval.enabled is set to true. The valid range of this config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).",
+      "defaultValue": "20"
+    },
+    {
+      "key": "spark.sql.repl.eagerEval.truncate",
+      "doc": "The max number of characters for each cell that is returned by eager evaluation. This only takes effect when spark.sql.repl.eagerEval.enabled is set to true.",
+      "defaultValue": "20"
+    },
+    {
+      "key": "spark.sql.requireAllClusterKeysForCoPartition",
+      "doc": "When true, the planner requires all the clustering keys as the hash partition keys of the children, to eliminate the shuffles for the operator that needs its children to be co-partitioned, such as JOIN node. This is to avoid data skews which can lead to significant performance regression if shuffles are eliminated.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.requireAllClusterKeysForDistribution",
+      "doc": "When true, the planner requires all the clustering keys as the partition keys (with same ordering) of the children, to eliminate the shuffle for the operator that requires its children be clustered distributed, such as AGGREGATE and WINDOW node. This is to avoid data skews which can lead to significant performance regression if shuffle is eliminated.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.resultQueryStage.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to execute ResultQueryStageExec in AQE",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.retainGroupColumns",
+      "doc": "",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.runCollationTypeCastsBeforeAliasAssignment.enabled",
+      "doc": "When set to true, rules like ResolveAliases or ResolveAggregateFunctions will run CollationTypeCasts before alias assignment. This is necessary for correct alias generation.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.runSQLOnFiles",
+      "doc": "When true, we could use `datasource`.`path` as table in SQL query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.scriptTransformation.exitTimeoutInSeconds",
+      "doc": "Timeout for executor to wait for the termination of transformation script when EOF.",
+      "defaultValue": "10000ms"
+    },
+    {
+      "key": "spark.sql.scripting.continueHandlerEnabled",
+      "doc": "SQL Scripting CONTINUE HANDLER feature enables CONTINUE exception handlers which are essential for cursor iteration with NOT FOUND conditions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.scripting.cursorEnabled",
+      "doc": "SQL Scripting CURSOR feature enables declarative cursors with DECLARE CURSOR, OPEN, FETCH, and CLOSE statements.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.scripting.enabled",
+      "doc": "SQL Scripting feature is under development and its use should be done under this feature flag. SQL Scripting enables users to write procedural SQL including control flow and error handling.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.selfJoinAutoResolveAmbiguity",
+      "doc": "",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.session.localRelationBatchOfChunksSizeBytes",
+      "doc": "Limit on how much memory the client can use when uploading a local relation to the server. The client collects multiple local relation chunks into a single batch in memory until the limit is reached, then uploads the batch to the server. This helps reduce memory pressure on the client when dealing with very large local relations because the client does not have to materialize all chunks in memory. Limits the spark.sql.session.localRelationChunkSizeBytes, a minimum of the two confs is used to determine the chunk size.",
+      "defaultValue": "1073741824"
+    },
+    {
+      "key": "spark.sql.session.localRelationCacheThreshold",
+      "doc": "The threshold for the size in bytes of local relations to be cached at the driver side after serialization.",
+      "defaultValue": "1048576"
+    },
+    {
+      "key": "spark.sql.session.localRelationChunkSizeBytes",
+      "doc": "The chunk size in bytes when splitting ChunkedCachedLocalRelation.data into batches. A new chunk is created when either spark.sql.session.localRelationChunkSizeBytes or spark.sql.session.localRelationChunkSizeRows is reached. Limited by the spark.sql.session.localRelationBatchOfChunksSizeBytes, a minimum of the two confs is used to determine the chunk size.",
+      "defaultValue": "16777216"
+    },
+    {
+      "key": "spark.sql.session.localRelationChunkSizeLimit",
+      "doc": "Limit on how large a single chunk of a ChunkedCachedLocalRelation.data can be in bytes. If the limit is exceeded, an exception is thrown.",
+      "defaultValue": "2000MB"
+    },
+    {
+      "key": "spark.sql.session.localRelationChunkSizeRows",
+      "doc": "The chunk size in number of rows when splitting ChunkedCachedLocalRelation.data into batches. A new chunk is created when either spark.sql.session.localRelationChunkSizeBytes or spark.sql.session.localRelationChunkSizeRows is reached.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.session.localRelationSizeLimit",
+      "doc": "Limit on how large ChunkedCachedLocalRelation.data can be in bytes.If the limit is exceeded, an exception is thrown.",
+      "defaultValue": "3221225472"
+    },
+    {
+      "key": "spark.sql.session.timeZone",
+      "doc": "The ID of session local timezone in the format of either region-based zone IDs or zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format '(+|-)HH', '(+|-)HH:mm' or '(+|-)HH:mm:ss', e.g '-08', '+01:00' or '-13:33:33'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous.",
+      "defaultValue": null
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.in.memory.threshold",
+      "doc": "Threshold for number of windows guaranteed to be held in memory by the session window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.spill.size.threshold",
+      "doc": "Threshold for size of rows to be spilled by window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "fallback": "spark.shuffle.spill.maxSizeInBytesForSpillThreshold"
+    },
+    {
+      "key": "spark.sql.sessionWindow.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by window operator. Note that the buffer is used only for the query Spark cannot apply aggregations on determining session window.",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.shuffle.orderIndependentChecksum.enableFullRetryOnMismatch",
+      "doc": "Whether to retry all tasks of a consumer stage when we detect checksum mismatches with its producer stages.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.shuffle.orderIndependentChecksum.enableQueryLevelRollbackOnMismatch",
+      "doc": "Whether to rollback all the consumer stages from the same query executor when we detect checksum mismatches with its producer stages, including cancel running shuffle map stages and resubmit, clean up all the shuffle data written for available shuffle map stages and abort the running result stages.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.shuffle.orderIndependentChecksum.enabled",
+      "doc": "Whether to calculate order independent checksum for the shuffle data or not. If enabled, Spark will calculate a checksum that is independent of the input row order for each mapper and returns the checksums from executors to driver. This is different from the checksum computed when spark.shuffle.checksum.enabled is enabled which is sensitive to shuffle data ordering to detect file corruption. While this checksum will be the same even if the shuffle row order changes and it is used to detect whether different task attempts of the same partition produce different output data or not (same set of keyValue pairs). In case the output data has changed across retries, Spark will need to retry all tasks of the consumer stages to avoid correctness issues.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.shuffle.partitions",
+      "doc": "The default number of partitions to use when shuffling data for joins or aggregations.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.shuffleDependency.fileCleanup.enabled",
+      "doc": "(Deprecated since Spark 4.1, please set 'spark.sql.connect.shuffleDependency.fileCleanup.enabled'.)",
+      "defaultValue": "false",
+      "deprecated": {
+        "version": "4.1",
+        "comment": "Use 'spark.sql.connect.shuffleDependency.fileCleanup.enabled' instead."
+      }
+    },
+    {
+      "key": "spark.sql.shuffleDependency.skipMigration.enabled",
+      "doc": "When enabled, shuffle dependencies for a Spark Connect SQL execution are marked at the end of the execution, and they will not be migrated during decommissions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.shuffleExchange.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism for doing preparation of shuffle exchange, which includes subquery execution, file listing, etc.",
+      "defaultValue": "1024",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.shuffledHashJoinFactor",
+      "doc": "The shuffle hash join can be selected if the data size of small side multiplied by this factor is still smaller than the large side.",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.sort.enableRadixSort",
+      "doc": "When true, enable use of radix sort when possible. Radix sort is much faster but requires additional memory to be reserved up-front. The memory overhead may be significant when sorting very small rows (up to 50% more in this case).",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the sort merge join operator",
+      "defaultValue": "2147483632"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.spill.size.threshold",
+      "doc": "Threshold for size of rows to be spilled by sort merge join operator",
+      "fallback": "spark.shuffle.spill.maxSizeInBytesForSpillThreshold"
+    },
+    {
+      "key": "spark.sql.sortMergeJoinExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by sort merge join operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.sources.binaryFile.maxLength",
+      "doc": "The max length of a file that can be read by the binary file data source. Spark will fail fast and not attempt to read the file if its length exceeds this value. The theoretical max is Int.MaxValue, though VMs might implement a smaller max.",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.autoBucketedScan.enabled",
+      "doc": "When true, decide whether to do bucketed scan on input tables based on query plan automatically. Do not use bucketed scan if 1. query does not have operators to utilize bucketing (e.g. join, group-by, etc), or 2. there's an exchange operator between these operators and table scan. Note when 'spark.sql.sources.bucketing.enabled' is set to false, this configuration does not take any effect.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.enabled",
+      "doc": "When false, we will treat bucketed table as normal table",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.bucketing.maxBuckets",
+      "doc": "The maximum number of buckets allowed.",
+      "defaultValue": "100000"
+    },
+    {
+      "key": "spark.sql.sources.commitProtocolClass",
+      "doc": "",
+      "defaultValue": "org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol"
+    },
+    {
+      "key": "spark.sql.sources.default",
+      "doc": "The default data source to use in input/output.",
+      "defaultValue": "parquet"
+    },
+    {
+      "key": "spark.sql.sources.disabledJdbcConnProviderList",
+      "doc": "Configures a list of JDBC connection providers, which are disabled. The list contains the name of the JDBC connection providers separated by comma.",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.sources.fileCompressionFactor",
+      "doc": "When estimating the output data size of a table scan, multiply the file size with this factor as the estimated data size, in case the data is compressed in the file and lead to a heavily underestimated result.",
+      "defaultValue": "1.0"
+    },
+    {
+      "key": "spark.sql.sources.ignoreDataLocality",
+      "doc": "If true, Spark will not fetch the block locations for each file on listing files. This speeds up file listing, but the scheduler cannot schedule tasks to take advantage of data locality. It can be particularly useful if data is read from a remote cluster so the scheduler could never take advantage of locality anyway.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.outputCommitterClass",
+      "doc": ""
+    },
+    {
+      "key": "spark.sql.sources.parallelPartitionDiscovery.parallelism",
+      "doc": "The number of parallelism to list a collection of path recursively, Set the number to prevent file listing from generating too many tasks.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.sources.parallelPartitionDiscovery.threshold",
+      "doc": "The maximum number of paths allowed for listing files at driver side. If the number of detected paths exceeds this value during partition discovery, it tries to list the files with another Spark distributed job. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.",
+      "defaultValue": "32"
+    },
+    {
+      "key": "spark.sql.sources.partitionColumnTypeInference.enabled",
+      "doc": "When true, automatically infer the data types for partitioned columns.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.partitionOverwriteMode",
+      "doc": "When INSERT OVERWRITE a partitioned data source table, we currently support 2 modes: static and dynamic. In static mode, Spark deletes all the partitions that match the partition specification(e.g. PARTITION(a=1,b)) in the INSERT statement, before overwriting. In dynamic mode, Spark doesn't delete partitions ahead, and only overwrite those partitions that have data written into it at runtime. By default we use static mode to keep the same behavior of Spark prior to 2.3. Note that this config doesn't affect Hive serde tables, as they are always overwritten with dynamic mode. This can also be set as an output option for a data source using key partitionOverwriteMode (which takes precedence over this setting), e.g. dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path).",
+      "defaultValue": "STATIC"
+    },
+    {
+      "key": "spark.sql.sources.schemaStringLengthThreshold",
+      "doc": "The maximum length allowed in a single cell when storing additional schema information in Hive's metastore.",
+      "defaultValue": "4000",
+      "isStatic": true,
+      "deprecated": {
+        "version": "3.2",
+        "comment": "Use 'spark.sql.hive.tablePropertyLengthThreshold' instead."
+      }
+    },
+    {
+      "key": "spark.sql.sources.useListFilesFileSystemList",
+      "doc": "A comma-separated list of file system schemes to use FileSystem.listFiles API for a single root path listing",
+      "defaultValue": "s3a"
+    },
+    {
+      "key": "spark.sql.sources.useV1SourceList",
+      "doc": "A comma-separated list of data source short names or fully qualified data source implementation class names for which Data Source V2 code path is disabled. These data sources will fallback to Data Source V1 code path.",
+      "defaultValue": "avro,csv,json,kafka,orc,parquet,text"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.allowCompatibleTransforms.enabled",
+      "doc": "Whether to allow storage-partition join in the case where the partition transforms are compatible but not identical.  This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled and spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled to be disabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.allowJoinKeysSubsetOfPartitionKeys.enabled",
+      "doc": "Whether to allow storage-partition join in the case where join keys are a subset of the partition keys of the source tables. At planning time, Spark will group the partitions by only those keys that are in the join keys. This is currently enabled only if spark.sql.requireAllClusterKeysForDistribution is false.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.enabled",
+      "doc": "Similar to spark.sql.sources.bucketing.enabled, this config is used to enable bucketing for V2 data sources. When turned on, Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and will try to avoid shuffle if necessary.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled",
+      "doc": "During a storage-partitioned join, whether to allow input partitions to be partially clustered, when both sides of the join are of KeyedPartitioning. At planning time, Spark will pick the side with less data size based on table statistics, group and replicate them to match the other side. This is an optimization on skew join and can help to reduce data skewness when certain partitions are assigned large amount of data. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.partition.filter.enabled",
+      "doc": "Whether to filter partitions when running storage-partition join. When enabled, partitions without matches on the other side can be omitted for scanning, if allowed by the join type. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.partitionKeyOrdering.enabled",
+      "doc": "When enabled, Spark derives output ordering from the partition key expressions of a V2 data source that reports a KeyedPartitioning but does not report explicit ordering via SupportsReportOrdering. Within a single partition all rows share the same key value, so the data is trivially sorted by those expressions. Requires spark.sql.sources.v2.bucketing.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.preserveKeyOrderingOnCoalesce.enabled",
+      "doc": "When enabled, Spark preserves sort orders over partition key expressions when GroupPartitionsExec coalesces multiple input partitions into one output partition. Because all merged partitions share the same partition key value, sort orders over those key expressions remain valid after the merge. This applies to both key-derived ordering (from SupportsReportOrdering) and ordering derived from spark.sql.sources.v2.bucketing.partitionKeyOrdering.enabled. Requires spark.sql.sources.v2.bucketing.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.preserveOrderingOnCoalesce.enabled",
+      "doc": "When turned on, GroupPartitionsExec will use sorted merge to preserve full ordering (as opposed to the key-derived ordering preserved by spark.sql.sources.v2.bucketing.preserveKeyOrderingOnCoalesce.enabled) when coalescing multiple partitions with the same key. This allows eliminating downstream sorts when data is both partitioned and sorted. When this config is enabled, the effect of spark.sql.sources.v2.bucketing.preserveKeyOrderingOnCoalesce.enabled is fully subsumed: full ordering implies key-derived ordering. However, sorted merge uses more resources (priority queue, comparison overhead) than simple concatenation, especially when coalescing many partitions. When turned off, only key-derived ordering is preserved during coalescing. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.pushPartValues.enabled",
+      "doc": "Whether to pushdown common partition values when spark.sql.sources.v2.bucketing.enabled is enabled. When turned on, if both sides of a join are of KeyedPartitioning and if they share compatible partition keys, even if they don't have the exact same partition values, Spark will calculate a superset of partition values and pushdown that info to group partition nodes, which will use empty partitions for the missing partition values on either side. This could help to eliminate unnecessary shuffles",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.shuffle.enabled",
+      "doc": "During a storage-partitioned join, whether to allow to shuffle only one side. When only one side is KeyedPartitioning, if the conditions are met, spark will only shuffle the other side. This optimization will reduce the amount of data that needs to be shuffle. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.v2.bucketing.sorting.enabled",
+      "doc": "When turned on, Spark will recognize the specific distribution reported by a V2 data source through SupportsReportPartitioning, and will try to avoid a shuffle if possible when sorting by those columns. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.sources.validatePartitionColumns",
+      "doc": "When this option is set to true, partition column values will be validated with user-specified schema. If the validation fails, a runtime exception is thrown. When this option is set to false, the partition column value will be converted to null if it can not be casted to corresponding user-specified schema.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.stableDerivedColumnAlias.enabled",
+      "doc": "Enable deriving of stable column aliases from the lexer tree instead of parse tree and form them via pretty SQL print.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.stackTracesInDataFrameContext",
+      "doc": "The number of non-Spark stack traces in the captured DataFrame query context.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.statistics.fallBackToHdfs",
+      "doc": "When true, it will fall back to HDFS if the table statistics are not available from table metadata. This is useful in determining if a table is small enough to use broadcast joins. This flag is effective only for non-partitioned Hive tables. For non-partitioned data source tables, it will be automatically recalculated if table statistics are not available. For partitioned data source and partitioned Hive tables, It is 'spark.sql.defaultSizeInBytes' if table statistics are not available.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.histogram.enabled",
+      "doc": "Generates histograms when computing column statistics if enabled. Histograms can provide better estimation accuracy. Currently, Spark only supports equi-height histogram. Note that collecting histograms takes extra cost. For example, collecting column statistics usually takes only one table scan, but generating equi-height histogram will cause an extra table scan.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.histogram.numBins",
+      "doc": "The number of bins when generating histograms.",
+      "defaultValue": "254"
+    },
+    {
+      "key": "spark.sql.statistics.ndv.maxError",
+      "doc": "The maximum relative standard deviation allowed in HyperLogLog++ algorithm when generating column level statistics.",
+      "defaultValue": "0.05"
+    },
+    {
+      "key": "spark.sql.statistics.parallelFileListingInStatsComputation.enabled",
+      "doc": "When true, SQL commands use parallel file listing, as opposed to single thread listing. This usually speeds up commands that need to list many directories.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.statistics.percentile.accuracy",
+      "doc": "Accuracy of percentile approximation when generating equi-height histograms. Larger value means better accuracy. The relative error can be deduced by 1.0 / PERCENTILE_ACCURACY.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.statistics.size.autoUpdate.enabled",
+      "doc": "Enables automatic update for table size once table's data is changed. Note that if the total number of files of the table is very large, this can be expensive and slow down data change commands.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.statistics.updatePartitionStatsInAnalyzeTable.enabled",
+      "doc": "When this config is enabled, Spark will also update partition statistics in analyze table command (i.e., ANALYZE TABLE .. COMPUTE STATISTICS [NOSCAN]). Note the command will also become more expensive. When this config is disabled, Spark will only update table level statistics.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.storeAssignmentPolicy",
+      "doc": "When inserting a value into a column with different data type, Spark will perform type coercion. Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy, Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL. It disallows certain unreasonable type conversions such as converting `string` to `int` or `double` to `boolean`. With legacy policy, Spark allows the type coercion as long as it is a valid `Cast`, which is very loose. e.g. converting `string` to `int` or `double` to `boolean` is allowed. It is also the only behavior in Spark 2.x and it is compatible with Hive. With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is not allowed.",
+      "defaultValue": "ANSI"
+    },
+    {
+      "key": "spark.sql.streaming.aggregation.stateFormatVersion",
+      "doc": "State format version used by streaming aggregation operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.asyncLogPurge.enabled",
+      "doc": "When true, purging the offset log and commit log of old entries will be done asynchronously.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkUnfinishedRepartitionOnRestart",
+      "doc": "When true, checks if the latest batch is for an unfinished state repartitioning operation, on query restart. If so, it will fail the query and require the repartitioning to be complete before restarting the query.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.escapedPathCheck.enabled",
+      "doc": "Whether to detect a streaming query may pick up an incorrect checkpoint path due to SPARK-26824.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.fileChecksum.enabled",
+      "doc": "When true, checksum would be generated and verified for checkpoint files. This is used to detect file corruption. This is only enabled when STATE_STORE_CHECKPOINT_FORMAT_VERSION >= 2",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.fileChecksum.skipCreationIfFileMissingChecksum",
+      "doc": "When true, if a microbatch is retried, if a file already exists but its checksum file does not exist, the file checksum will not be created. This is useful for compatibility with files created before file checksums were enabled.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.renamedFileCheck.enabled",
+      "doc": "When true, Spark will validate if renamed checkpoint file exists.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.checkpoint.verifyMetadataExists.enabled",
+      "doc": "When true, validates that the checkpoint metadata file exists when offset or commit logs contain data. This prevents generating a new query ID when checkpoint data already exists, which would cause data duplication in exactly-once sinks.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.checkpointLocation",
+      "doc": "The default location for storing checkpoint data for streaming queries."
+    },
+    {
+      "key": "spark.sql.streaming.commitProtocolClass",
+      "doc": "",
+      "defaultValue": "org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.epochBacklogQueueSize",
+      "doc": "The max number of entries to be stored in queue to wait for late epochs. If this parameter is exceeded by the size of the queue, stream will stop with an error.",
+      "defaultValue": "10000"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.executorPollIntervalMs",
+      "doc": "The interval at which continuous execution readers will poll to check whether the epoch has advanced on the driver.",
+      "defaultValue": "100ms"
+    },
+    {
+      "key": "spark.sql.streaming.continuous.executorQueueSize",
+      "doc": "The size (measured in number of rows) of the queue used in continuous execution to buffer the results of a ContinuousDataReader.",
+      "defaultValue": "1024"
+    },
+    {
+      "key": "spark.sql.streaming.disabledV2MicroBatchReaders",
+      "doc": "A comma-separated list of fully qualified data source register class names for which MicroBatchReadSupport is disabled. Reads from these sources will fall back to the V1 Sources.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.streaming.disabledV2Writers",
+      "doc": "A comma-separated list of fully qualified data source register class names for which StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.cleanupDelay",
+      "doc": "How long that a file is guaranteed to be visible for all readers.",
+      "defaultValue": "600000ms"
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.compactInterval",
+      "doc": "Number of log files after which all the previous files are compacted into the next log file.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.fileSink.log.deletion",
+      "doc": "Whether to delete the expired log files in file stream sink.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.cleaner.numThreads",
+      "doc": "Number of threads used in the file source completed file cleaner.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.cleanupDelay",
+      "doc": "How long in milliseconds a file is guaranteed to be visible for all readers.",
+      "defaultValue": "600000ms"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.compactInterval",
+      "doc": "Number of log files after which all the previous files are compacted into the next log file.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.log.deletion",
+      "doc": "Whether to delete the expired log files in file stream source.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileSource.schema.forceNullable",
+      "doc": "When true, force the schema of streaming file source to be nullable (including all the fields). Otherwise, the schema might not be compatible with actual data, which leads to corruptions.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.fileStreamSink.ignoreMetadata",
+      "doc": "If this is enabled, when Spark reads from the results of a streaming query written by `FileStreamSink`, Spark will ignore the metadata log and treat it as normal path to read, e.g. listing files using HDFS APIs.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.flatMapGroupsWithState.skipEmittingInitialStateKeys",
+      "doc": "When true, the flatMapGroupsWithState operation in a streaming query will not emit results for the initial state keys of each group.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion",
+      "doc": "State format version used by flatMapGroupsWithState operation in a streaming query",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.forceDeleteTempCheckpointLocation",
+      "doc": "When true, enable temporary checkpoint locations force delete.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.internal.stateStore.partitions",
+      "doc": "WARN: This config is used internally and is not intended to be user-facing. This config can be removed without support of compatibility in any time. DO NOT USE THIS CONFIG DIRECTLY AND USE THE CONFIG `spark.sql.shuffle.partitions`. The default number of partitions to use when shuffling data for stateful operations. If not specified, this config picks up the value of `spark.sql.shuffle.partitions`. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location."
+    },
+    {
+      "key": "spark.sql.streaming.join.stateFormatV4.enabled",
+      "doc": "When true, enables state format version 4 for stream-stream joins. This config will be removed once V4 is complete.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.join.stateFormatVersion",
+      "doc": "State format version used by streaming join operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running. Version 3 uses a single state store with virtual column families instead of four stores and is only supported with RocksDB. NOTE: version 1 is DEPRECATED and should not be explicitly set by users. Version 4 is under development and only available for testing.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.kafka.useDeprecatedOffsetFetching",
+      "doc": "When true, the deprecated Consumer based offset fetching used which could cause infinite wait in Spark queries. Such cases query restart is the only workaround. For further details please see Offset Fetching chapter of Structured Streaming Kafka Integration Guide.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.maxBatchesToRetainInMemory",
+      "doc": "The maximum number of batches which will be retained in memory to avoid loading from files. The value adjusts a trade-off between memory usage vs cache miss: '2' covers both success and direct failure cases, '1' covers only success case, and '0' covers extreme case - disable cache to maximize memory size of executors.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.metadataCache.enabled",
+      "doc": "Whether the streaming HDFSMetadataLog caches the metadata of the latest two batches.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.metricsEnabled",
+      "doc": "Whether Dropwizard/Codahale metrics will be reported for active streaming queries.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.minBatchesToRetain",
+      "doc": "The minimum number of batches that must be retained and made recoverable.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.streaming.multipleWatermarkPolicy",
+      "doc": "Policy to calculate the global watermark value when there are multiple watermark operators in a streaming query. The default value is 'min' which chooses the minimum watermark reported across multiple operators. Other alternative value is 'max' which chooses the maximum across multiple operators. Note: This configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "min"
+    },
+    {
+      "key": "spark.sql.streaming.noDataMicroBatches.enabled",
+      "doc": "Whether streaming micro-batch engine will execute batches without data for eager state management for stateful streaming queries.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.noDataProgressEventInterval",
+      "doc": "How long to wait before providing query idle event when there is no data",
+      "defaultValue": "10000ms"
+    },
+    {
+      "key": "spark.sql.streaming.numRecentProgressUpdates",
+      "doc": "The number of progress updates to retain for a streaming query",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.streaming.offsetLog.formatVersion",
+      "doc": "The version of the offset log format. The default value is 1.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.optimizeOneRowPlan.enabled",
+      "doc": "When true, enable OptimizeOneRowPlan rule for the case where the child is a streaming Dataset. This is a fallback flag to revert the 'incorrect' behavior, hence this configuration must not be used without understanding in depth. Use this only to quickly recover failure in existing query!",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.pollingDelay",
+      "doc": "How long to delay polling new data when no data is available",
+      "defaultValue": "10ms"
+    },
+    {
+      "key": "spark.sql.streaming.queryEvolution.enableSinkEvolution",
+      "doc": "When true, streaming sinks can be named using the name() API on DataStreamWriter. This enables sink evolution capability where sinks can be changed while maintaining a historical record of all sinks used in the checkpoint.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.queryEvolution.enableSourceEvolution",
+      "doc": "When true, enforces that all streaming sources must be explicitly named via the .name() API. This enables streaming source evolution, allowing sources to be added, removed, or reordered without losing state.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.ratioExtraSpaceAllowedInCheckpoint",
+      "doc": "The ratio of extra space allowed for batch deletion of files when maintenance isinvoked. When value > 0, it optimizes the cost of discovering and deleting old checkpoint versions. The minimum number of stale versions we retain in checkpoint location for batch deletion is calculated by minBatchesToRetain * ratioExtraSpaceAllowedInCheckpoint.",
+      "defaultValue": "0.3"
+    },
+    {
+      "key": "spark.sql.streaming.realTimeMode.allowlistCheck",
+      "doc": "Whether to check all operators, sinks used in real-time mode are in the allowlist.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.realTimeMode.minBatchDuration",
+      "doc": "The minimum long-running batch duration in milliseconds for real-time mode.",
+      "defaultValue": "5000ms"
+    },
+    {
+      "key": "spark.sql.streaming.schemaInference",
+      "doc": "Whether file-based streaming sources will infer its own schema",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.sessionWindow.merge.sessions.in.local.partition",
+      "doc": "When true, streaming session window sorts and merge sessions in local partition prior to shuffle. This is to reduce the rows to shuffle, but only beneficial when there're lots of rows in a batch being assigned to same sessions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.sessionWindow.stateFormatVersion",
+      "doc": "State format version used by streaming session window in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.autoSnapshotRepair.enabled",
+      "doc": "When true, enables automatic repair of state store snapshot, when a bad snapshot is detected while loading the state store, to prevent the query from failing. Typically, queries will fail when they are unable to load a snapshot, but this helps recover by skipping the bad snapshot and uses the change files.NOTE: For RocksDB state store, changelog checkpointing must be enabled",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.autoSnapshotRepair.maxChangeFileReplay",
+      "doc": "When autoSnapshotRepair is enabled, this specifies the maximum number of change files allowed to be replayed to rebuild state due to bad snapshots.",
+      "defaultValue": "500"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.autoSnapshotRepair.numFailuresBeforeActivating",
+      "doc": "When autoSnapshotRepair is enabled, it will wait for the specified number of snapshot load failures, before it attempts to repair.",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.checkpointFormatVersion",
+      "doc": "The version of the approach of doing state store checkpoint",
+      "defaultValue": "1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.commitValidation.enabled",
+      "doc": "When true, Spark will validate that all StateStore instances have committed for stateful streaming queries using foreachBatch. This helps detect cases where user-defined functions in foreachBatch (e.g., show(), limit()) don't process all partitions, which can lead to incorrect results. The validation only applies to foreachBatch sinks without global aggregates or limits.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.compression.codec",
+      "doc": "The codec used to compress delta and snapshot files generated by StateStore. By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify the codec. Default codec is lz4.",
+      "defaultValue": "lz4"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.coordinatorReportSnapshotUploadLag",
+      "doc": "When enabled, the state store coordinator will report state stores whose snapshot have not been uploaded for some time. See the conf snapshotLagReportInterval for the minimum time between reports, and the conf multiplierForMinVersionDiffToLog and multiplierForMinTimeDiffToLog for the logging thresholds.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.encodingFormat",
+      "doc": "The encoding format used for stateful operators to store information in the state store",
+      "defaultValue": "unsaferow"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.fileChecksumThreadPoolSize",
+      "doc": "Number of threads used to read/write files and their corresponding checksum files concurrently. Set to 0 to disable the thread pool and run operations sequentially on the calling thread. WARNING: Reducing below the default value of 4 may have performance impact.",
+      "defaultValue": "4"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.forceSnapshotUploadOnLag",
+      "doc": "When enabled, state stores with lagging snapshot uploads will automatically trigger a snapshot on the next commit. Requires spark.sql.streaming.stateStore.coordinatorReportSnapshotUploadLag to be true.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.formatValidation.enabled",
+      "doc": "When true, check if the data from state store is valid or not when running streaming queries. This can happen if the state store format has been changed. Note, the feature is only effective in the build-in HDFS state store provider now.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maintenanceForceShutdownTimeout",
+      "doc": "Timeout in seconds to wait for tasks to respond to cancellation after force shutdown is initiated. This applies after the graceful shutdown timeout has been exceeded.",
+      "defaultValue": "60000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maintenanceInterval",
+      "doc": "The interval in milliseconds between triggering maintenance tasks in StateStore. The maintenance task executes background maintenance task in all the loaded store providers if they are still the active instances according to the coordinator. If not, inactive instances of store providers will be closed.",
+      "defaultValue": "60000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maintenanceProcessingTimeout",
+      "doc": "Timeout in seconds to wait for maintenance to process this partition.",
+      "defaultValue": "30000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maintenanceShutdownTimeout",
+      "doc": "Timeout in seconds for maintenance pool operations to complete on shutdown",
+      "defaultValue": "300000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maxLaggingStoresToReport",
+      "doc": "Maximum number of state stores the coordinator will report as trailing in snapshot uploads. Stores are selected based on the most lagging behind in snapshot version.",
+      "defaultValue": "5"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maxNumStateSchemaFiles",
+      "doc": "The maximum number of StateSchemaV3 files allowed per operator",
+      "defaultValue": "128"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.maxVersionsToDeletePerMaintenance",
+      "doc": "The maximum number of versions to delete per maintenance operation. By default, this value is set to -1, which means no limit. Note that, currently this is only supported for the RocksDB state store provider.",
+      "defaultValue": "-1"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.minDeltasForSnapshot",
+      "doc": "Minimum number of state store delta files that needs to be generated before they consolidated into snapshots.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.multiplierForMinTimeDiffToLog",
+      "doc": "Determines the time threshold for logging warnings when a state store falls behind. The coordinator logs a warning when the store's uploaded snapshot timestamp trails the current time by the configured maintenance interval, times this multiplier.",
+      "defaultValue": "10"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.multiplierForMinVersionDiffToLog",
+      "doc": "Determines the version threshold for logging warnings when a state store falls behind. The coordinator logs a warning when the store's uploaded snapshot version trails the query's latest version by the configured number of deltas needed to create a snapshot, times this multiplier.",
+      "defaultValue": "5"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.numStateStoreInstanceMetricsToReport",
+      "doc": "Number of state store instance metrics included in streaming query progress messages per stateful operator. Instance metrics are selected based on metric-specific ordering to minimize noise in the progress report.",
+      "defaultValue": "5"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.numStateStoreMaintenanceThreads",
+      "doc": "Number of threads in the thread pool that perform clean up and snapshotting tasks for stateful streaming queries. The default value is the number of cores * 0.25 so that this thread pool doesn't take too many resources away from the query and affect performance.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.providerClass",
+      "doc": "The class used to manage state data in stateful streaming queries. This class must be a subclass of StateStoreProvider, and must have a zero-arg constructor. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.rocksdb.formatVersion",
+      "doc": "Set the RocksDB format version. This will be stored in the checkpoint when starting a streaming query. The checkpoint will use this RocksDB format version in the entire lifetime of the query.",
+      "defaultValue": "5"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.rocksdb.mergeOperatorVersion",
+      "doc": "Set the RocksDB merge operator version. This will be stored in the checkpoint when starting a streaming query. The checkpoint will use this merge operator version for the entire lifetime of the query.",
+      "defaultValue": "2"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.rowChecksum.enabled",
+      "doc": "When true, checksum would be generated and verified for each state store row. This is used to detect row level corruption. Note: This configuration cannot be changed between query restarts from the same checkpoint location.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.rowChecksum.readVerificationRatio",
+      "doc": "When specified, Spark will do row checksum verification for every specified number of rows read from state store. The check is to ensure the row read from state store is not corrupt. Default is 0, which means no verification during read but we will still do verification when loading from checkpoint location.Example, if you set to 1, it will do the check for every row read from the state store.If set to 10, it will do the check for every 10th row read from the state store.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.skipNullsForStreamStreamJoins.enabled",
+      "doc": "When true, this config will skip null values in hash based stream-stream joins. The number of skipped null values will be shown as custom metric of stream join operator. If the streaming query was started with Spark 3.5 or above, please exercise caution before enabling this config since it may hide potential data loss/corruption issues.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.snapshotLagReportInterval",
+      "doc": "The minimum amount of time between the state store coordinator's reports on state store instances trailing behind in snapshot uploads.",
+      "defaultValue": "300000ms"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.stateSchemaCheck",
+      "doc": "When true, Spark will validate the state schema against schema on existing state and fail query if it's incompatible.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.unloadOnCommit",
+      "doc": "When true, Spark will synchronously run maintenance and then close each StateStore instance on task completion. This removes the overhead of keeping every StateStore loaded indefinitely, at the cost of having to reload each StateStore every batch. Stateful applications that are failing due to resource exhaustion or that use dynamic allocation may benefit from enabling this.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.stateStore.valueStateSchemaEvolutionThreshold",
+      "doc": "The maximum number of value state schema evolutions allowed per column family",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.allowMultiple",
+      "doc": "When true, multiple stateful operators are allowed to be present in a streaming pipeline. The support for multiple stateful operators introduces a minor (semantically correct) change in respect to late record filtering - late records are detected and filtered in respect to the watermark from the previous microbatch instead of the current one. This is a behavior change for Spark streaming pipelines and we allow users to revert to the previous behavior of late record filtering (late records are detected and filtered by comparing with the current microbatch watermark) by setting the flag value to false. In this mode, only a single stateful operator will be allowed in a streaming pipeline.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.checkCorrectness.enabled",
+      "doc": "When true, the stateful operators for streaming query will be checked for possible correctness issue due to global watermark. The correctness issue comes from queries containing stateful operation which can emit rows older than the current watermark plus allowed late record delay, which are \"late rows\" in downstream stateful operations and these rows can be discarded. Please refer the programming guide doc for more details. Once the issue is detected, Spark will throw analysis exception. When this config is disabled, Spark will just print warning message for users. Prior to Spark 3.1.0, the behavior is disabling this config.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.statefulOperator.useStrictDistribution",
+      "doc": "The purpose of this config is only compatibility; DO NOT MANUALLY CHANGE THIS!!! When true, the stateful operator for streaming query will use StatefulOpClusteredDistribution which guarantees stable state partitioning as long as the operator provides consistent grouping keys across the lifetime of query. When false, the stateful operator for streaming query will use ClusteredDistribution which is not sufficient to guarantee stable state partitioning despite the operator provides consistent grouping keys across the lifetime of query. This config will be set to true for new streaming queries to guarantee stable state partitioning, and set to false for existing streaming queries to not break queries which are restored from existing checkpoints. Please refer SPARK-38204 for details.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stopActiveRunOnRestart",
+      "doc": "Running multiple runs of the same streaming query concurrently is not supported. If we find a concurrent active run for a streaming query (in the same or different SparkSessions on the same cluster) and this flag is true, we will stop the old streaming query run to start the new one.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.stopTimeout",
+      "doc": "How long to wait in milliseconds for the streaming execution thread to stop when calling the streaming query's stop() method. 0 or negative values wait indefinitely.",
+      "defaultValue": "0"
+    },
+    {
+      "key": "spark.sql.streaming.streamingQueryListeners",
+      "doc": "List of class names implementing StreamingQueryListener that will be automatically added to newly created sessions. The classes should have either a no-arg constructor, or a constructor that expects a SparkConf argument.",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.transformWithState.stateSchemaVersion",
+      "doc": "The version of the state schema used by the transformWithState operator",
+      "defaultValue": "3"
+    },
+    {
+      "key": "spark.sql.streaming.triggerAvailableNowWrapper.enabled",
+      "doc": "Whether to use the wrapper implementation of Trigger.AvailableNow if the source does not support Trigger.AvailableNow. Enabling this allows the benefits of Trigger.AvailableNow with sources which don't support it, but some sources may show unexpected behavior including duplication, data loss, etc. So use with extreme care! The ideal direction is to persuade developers of source(s) to support Trigger.AvailableNow.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.streaming.ui.enabled",
+      "doc": "Whether to run the Structured Streaming Web UI for the Spark application when the Spark Web UI is enabled.",
+      "defaultValue": "true",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.enabledCustomMetricList",
+      "doc": "Configures a list of custom metrics on Structured Streaming UI, which are enabled. The list contains the name of the custom metrics separated by comma. In aggregation only sum used. The list of supported custom metrics is state store provider specific and it can be found out for example from query progress log entry.",
+      "defaultValue": "",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.retainedProgressUpdates",
+      "doc": "The number of progress updates to retain for a streaming query for Structured Streaming UI.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.ui.retainedQueries",
+      "doc": "The number of inactive queries to retain for Structured Streaming UI.",
+      "defaultValue": "100",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.streaming.unsupportedOperationCheck",
+      "doc": "When true, the logical plan for streaming query will be checked for unsupported operations.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.validateEventTimeWatermarkColumn",
+      "doc": "When true, check that eventTime in withWatermark is a top-level column.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.streaming.verifyCheckpointDirectoryEmptyOnStart",
+      "doc": "When true, verifies that the checkpoint directory (offsets, state, commits) is empty when first starting a streaming query. This prevents prevents sharing checkpoint directories between different queries.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.cache.maxEntries",
+      "doc": "The maximum entries of the cache used for interpreted subexpression elimination.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.enabled",
+      "doc": "When true, common subexpressions will be eliminated.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.filterExec.enabled",
+      "doc": "When true (and subexpression elimination is enabled), FilterExec whole-stage codegen eliminates common subexpressions shared across its predicates. When false, FilterExec falls back to the predicate codegen that loads input columns lazily and short-circuits, avoiding eager materialization of all predicate-referenced columns on every row. Only affects FilterExec; subexpression elimination elsewhere is unaffected.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.subexpressionElimination.skipForShortcutExpr",
+      "doc": "When true, shortcut eliminate subexpression with `AND`, `OR`. The subexpression may not need to eval even if it appears more than once. e.g., `if(or(a, and(b, b)))`, the expression `b` would be skipped if `a` is true.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.subquery.maxThreadThreshold",
+      "doc": "The maximum degree of parallelism to execute the subquery.",
+      "defaultValue": "16",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.thriftServer.incrementalCollect",
+      "doc": "When true, enable incremental collection for execution in Thrift Server.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.thriftServer.interruptOnCancel",
+      "doc": "When true, all running tasks will be interrupted if one cancels a query. When false, all running tasks will remain until finished.",
+      "fallback": "spark.sql.execution.interruptOnCancel"
+    },
+    {
+      "key": "spark.sql.thriftServer.queryTimeout",
+      "doc": "Set a query duration timeout in seconds in Thrift Server. If the timeout is set to a positive value, a running query will be cancelled automatically when the timeout is exceeded, otherwise the query continues to run till completion. If timeout values are set for each statement via `java.sql.Statement.setQueryTimeout` and they are smaller than this configuration value, they take precedence. If you set this timeout and prefer to cancel the queries right away without waiting task to finish, consider enabling spark.sql.thriftServer.interruptOnCancel together.",
+      "defaultValue": "0ms"
+    },
+    {
+      "key": "spark.sql.thriftserver.scheduler.pool",
+      "doc": "Set a Fair Scheduler pool for a JDBC client session."
+    },
+    {
+      "key": "spark.sql.thriftserver.shuffleDependency.fileCleanup.enabled",
+      "doc": "When enabled, shuffle files will be cleaned up at the end of Thrift server SQL executions.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.thriftserver.ui.retainedSessions",
+      "doc": "The number of SQL client sessions kept in the JDBC/ODBC web UI history.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.thriftserver.ui.retainedStatements",
+      "doc": "The number of SQL statements kept in the JDBC/ODBC web UI history.",
+      "defaultValue": "200"
+    },
+    {
+      "key": "spark.sql.timeTravelTimestampKey",
+      "doc": "The option name to specify the time travel timestamp when reading a table.",
+      "defaultValue": "timestampAsOf"
+    },
+    {
+      "key": "spark.sql.timeTravelVersionKey",
+      "doc": "The option name to specify the time travel table version when reading a table.",
+      "defaultValue": "versionAsOf"
+    },
+    {
+      "key": "spark.sql.timeType.enabled",
+      "doc": "When true, the TIME data type is supported.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.timestampType",
+      "doc": "Configures the default timestamp type of Spark SQL, including SQL DDL, Cast clause, type literal and the schema inference of data sources. Setting the configuration as TIMESTAMP_NTZ will use TIMESTAMP WITHOUT TIME ZONE as the default type while putting it as TIMESTAMP_LTZ will use TIMESTAMP WITH LOCAL TIME ZONE. Before the 3.4.0 release, Spark only supports the TIMESTAMP WITH LOCAL TIME ZONE type.",
+      "defaultValue": "TIMESTAMP_LTZ"
+    },
+    {
+      "key": "spark.sql.transposeMaxValues",
+      "doc": "When doing a transpose without specifying values for the index column this is the maximum number of values that will be transposed without error.",
+      "defaultValue": "500"
+    },
+    {
+      "key": "spark.sql.truncateTable.ignorePermissionAcl.enabled",
+      "doc": "When set to true, TRUNCATE TABLE command will not try to set back original permission and ACLs when re-creating the table/partition paths.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.tvf.allowMultipleTableArguments.enabled",
+      "doc": "When true, allows multiple table arguments for table-valued functions, receiving the cartesian product of all the rows of these tables.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.types.framework.enabled",
+      "doc": "When true, use the Types Framework for supported types (currently TimeType). The framework centralizes type-specific operations in Ops classes instead of scattered pattern matching. When false, use legacy scattered implementation.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.udt.allowCreatingUDTFromString",
+      "doc": "When true, Spark loads and instantiates the UserDefinedType class named in a schema string (for example the schema stored in Parquet/ORC file metadata) while inferring or parsing a schema. Because the class name is taken from the data being read, a crafted file can make Spark load an arbitrary class from the classpath. Set this to false to block loading UDT classes by name, optionally allowing specific classes via 'spark.sql.udt.allowedDynamicUDTClasses'.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.udt.allowedDynamicUDTClasses",
+      "doc": "When 'spark.sql.udt.allowCreatingUDTFromString' is false, UserDefinedType classes listed here (by fully qualified class name) may still be loaded and instantiated from a schema string. Has no effect when UDT loading is enabled.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.ui.explainMode",
+      "doc": "Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.",
+      "defaultValue": "formatted"
+    },
+    {
+      "key": "spark.sql.ui.retainedExecutions",
+      "doc": "Number of executions to retain in the Spark UI.",
+      "defaultValue": "1000",
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.unionOutputPartitioning",
+      "doc": "When set to true, the output partitioning of UnionExec will be the same as the input partitioning if its children have same partitioning. Otherwise, it will be a default partitioning.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.useCommonExprIdForAlias",
+      "doc": "When true, use the common expression ID for the alias when rewriting With expressions. Otherwise, use the index of the common expression definition. When true this avoids duplicate alias names, but is helpful to set to false for testing to ensure that alias names are consistent.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variable.substitute",
+      "doc": "This enables substitution using syntax like `${var}`, `${system:var}`, and `${env:var}`.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variant.allowDuplicateKeys",
+      "doc": "When set to false, parsing variant from JSON will throw an error if there are duplicate keys in the input JSON object. When set to true, the parser will keep the last occurrence of all fields with the same key.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.variant.allowReadingShredded",
+      "doc": "When true, the Parquet reader is allowed to read shredded or unshredded variant. When false, it only reads unshredded variant.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variant.forceShreddingSchemaForTest",
+      "doc": "FOR INTERNAL TESTING ONLY. Sets shredding schema for Variant.",
+      "defaultValue": ""
+    },
+    {
+      "key": "spark.sql.variant.inferShreddingSchema",
+      "doc": "Infer shredding schema when writing Variant columns in Parquet tables.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variant.pushVariantIntoScan",
+      "doc": "When true, replace variant type in the scan schema with a struct containing requested fields.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variant.shredding.maxSchemaDepth",
+      "doc": "Maximum depth in Variant value to traverse when inferring a schema. Any array/object below this depth will be shredded as a single binary.",
+      "defaultValue": "50"
+    },
+    {
+      "key": "spark.sql.variant.shredding.maxSchemaWidth",
+      "doc": "Maximum number of shredded fields to create when inferring a schema for Variant",
+      "defaultValue": "300"
+    },
+    {
+      "key": "spark.sql.variant.validateUnicodeInJsonParsing",
+      "doc": "When true, parsing variant from JSON rejects strings that contain unpaired UTF-16 surrogate code units (such as a lone high surrogate like \\uD835), which are invalid per RFC 8259 section 7. When false, restores the legacy permissive behavior in which the unpaired surrogate is silently replaced by the Unicode replacement character during UTF-8 encoding, causing data corruption that diverges from strict JSON parsers.",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.variant.writeShredding.enabled",
+      "doc": "When true, the Parquet writer is allowed to write shredded variant. ",
+      "defaultValue": "true"
+    },
+    {
+      "key": "spark.sql.view.maxNestedViewDepth",
+      "doc": "The maximum depth of a view reference in a nested view. A nested view may reference other nested views, the dependencies are organized in a directed acyclic graph (DAG). However the DAG depth may become too large and cause unexpected behavior. This configuration puts a limit on this: when the depth of a view exceeds this value during analysis, we terminate the resolution to avoid potential errors.",
+      "defaultValue": "100"
+    },
+    {
+      "key": "spark.sql.warehouse.dir",
+      "doc": "The default location for managed databases and tables.",
+      "defaultValue": null,
+      "isStatic": true
+    },
+    {
+      "key": "spark.sql.window.segmentTree.blockSize",
+      "doc": "Block size, in rows, for the block-chunked segment tree used by moving window frames. Each leaf of the tree aggregates this many consecutive rows. Smaller values reduce per-partition memory and speed up tree build for small partitions, but make the tree deeper and increase query cost for wide frames. Larger values amortize build cost and shrink the tree but increase the per-block prefix/suffix scan cost within a block. The default is tuned for partitions on the order of tens of thousands to millions of rows.",
+      "defaultValue": "65536"
+    },
+    {
+      "key": "spark.sql.window.segmentTree.enabled",
+      "doc": "Use block-chunked segment tree for moving aggregate window frames whose functions are all DeclarativeAggregate without FILTER/DISTINCT.",
+      "defaultValue": "false"
+    },
+    {
+      "key": "spark.sql.window.segmentTree.fanout",
+      "doc": "Fanout of internal nodes for the block-chunked segment tree.",
+      "defaultValue": "16"
+    },
+    {
+      "key": "spark.sql.window.segmentTree.minPartitionRows",
+      "doc": "Minimum partition row count to activate the segment-tree moving frame. Partitions smaller than this fall back to the default sliding implementation.",
+      "defaultValue": "64"
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.in.memory.threshold",
+      "doc": "Threshold for number of rows guaranteed to be held in memory by the window operator",
+      "defaultValue": "4096"
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.spill.size.threshold",
+      "doc": "Threshold for size of rows to be spilled by window operator",
+      "fallback": "spark.shuffle.spill.maxSizeInBytesForSpillThreshold"
+    },
+    {
+      "key": "spark.sql.windowExec.buffer.spill.threshold",
+      "doc": "Threshold for number of rows to be spilled by window operator",
+      "defaultValue": "2147483647"
+    },
+    {
+      "key": "spark.sql.xml.variant.respectInferSchema",
+      "doc": "Kill switch for the SPARK-56554 fix. When true (default), the XML to Variant parser honors the 'inferSchema' option: if 'inferSchema' is false, primitive leaf values (text and attributes) are preserved as strings inside the Variant instead of being inferred as boolean, long, or decimal. Set this conf to false to restore the pre-SPARK-56554 behavior of always inferring types regardless of the 'inferSchema' option.",
+      "defaultValue": "true"
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/proto/spark/connect/base.proto b/crates/sail-spark-connect/proto/spark/connect/base.proto
index a97d2d25f..c7247129f 100644
--- a/crates/sail-spark-connect/proto/spark/connect/base.proto
+++ b/crates/sail-spark-connect/proto/spark/connect/base.proto
@@ -491,6 +491,12 @@ message ExecutePlanResponse {
     repeated Expression.Literal values = 2;
     repeated string keys = 3;
     int64 plan_id = 4;
+    // (Optional) The index of the root error in errors.
+    // The field will not be set if there are no errors.
+    optional int32 root_error_idx = 5;
+    // A list of errors that occurred while collecting the observed metrics.
+    // If the length is 0, it means no errors occurred.
+    repeated FetchErrorDetailsResponse.Error errors = 6;
   }
 
   message ResultComplete {
@@ -1213,11 +1219,94 @@ message CloneSessionResponse {
 
   // Session id of the new cloned session.
   string new_session_id = 3;
-  
+
   // Server-side session ID of the new cloned session.
   string new_server_side_session_id = 4;
 }
 
+// Next ID: 6
+message GetStatusRequest {
+  // (Required)
+  //
+  // The session_id specifies a Spark session for a user identified by user_context.user_id.
+  // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff`
+  string session_id = 1;
+
+  // (Required)
+  //
+  // user_context.user_id and session_id both identify a unique remote spark session on the
+  // server side.
+  UserContext user_context = 2;
+
+  // (Optional)
+  //
+  // Provides optional information about the client sending the request. This field
+  // can be used for language or version specific information and is only intended for
+  // logging purposes and will not be interpreted by the server.
+  optional string client_type = 3;
+
+  // (Optional)
+  //
+  // Server-side generated idempotency key from the previous responses (if any). Server
+  // can use this to validate that the server side session has not changed.
+  optional string client_observed_server_side_session_id = 4;
+
+  // (Optional)
+  //
+  // Get status of operations in the session.
+  optional OperationStatusRequest operation_status = 5;
+
+  // Extension point for custom status request types.
+  repeated google.protobuf.Any extensions = 999;
+
+  message OperationStatusRequest {
+    // Get status of operations with these operation_ids.
+    // If unset or empty, returns status of all operations in the session.
+    repeated string operation_ids = 1;
+
+    // Extension point for custom operation-level status requests.
+    repeated google.protobuf.Any extensions = 999;
+  }
+}
+
+// Next ID: 4
+message GetStatusResponse {
+  // Session id of the session for which the status was requested.
+  string session_id = 1;
+
+  // Server-side generated idempotency key that the client can use to assert that the server side
+  // session has not changed.
+  string server_side_session_id = 2;
+
+  // Status information about requested operations.
+  repeated OperationStatus operation_statuses = 3;
+
+  // Extension point for custom status response types.
+  repeated google.protobuf.Any extensions = 999;
+
+  // Status information for a single operation.
+  message OperationStatus {
+    // The operation_id of the operation.
+    string operation_id = 1;
+
+    // The current status of the operation.
+    OperationState state = 2;
+
+    // Extension point for custom operation-level status fields.
+    repeated google.protobuf.Any extensions = 999;
+
+    enum OperationState {
+      OPERATION_STATE_UNSPECIFIED = 0;
+      OPERATION_STATE_UNKNOWN = 1;
+      OPERATION_STATE_RUNNING = 2;
+      OPERATION_STATE_TERMINATING = 3;
+      OPERATION_STATE_SUCCEEDED = 4;
+      OPERATION_STATE_FAILED = 5;
+      OPERATION_STATE_CANCELLED = 6;
+    }
+  }
+}
+
 // Main interface for the SparkConnect service.
 service SparkConnectService {
 
@@ -1272,4 +1361,7 @@ service SparkConnectService {
   // The request can optionally specify a custom session ID for the cloned session (must be
   // a valid UUID). If not provided, a new UUID will be generated automatically.
   rpc CloneSession(CloneSessionRequest) returns (CloneSessionResponse) {}
+
+  // Get status information of different types.
+  rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {}
 }
diff --git a/crates/sail-spark-connect/proto/spark/connect/catalog.proto b/crates/sail-spark-connect/proto/spark/connect/catalog.proto
index 5b1b90b00..b5341d16e 100644
--- a/crates/sail-spark-connect/proto/spark/connect/catalog.proto
+++ b/crates/sail-spark-connect/proto/spark/connect/catalog.proto
@@ -55,6 +55,16 @@ message Catalog {
     CurrentCatalog current_catalog = 24;
     SetCurrentCatalog set_current_catalog = 25;
     ListCatalogs list_catalogs = 26;
+    DropTable drop_table = 27;
+    DropView drop_view = 28;
+    CreateDatabase create_database = 29;
+    DropDatabase drop_database = 30;
+    ListPartitions list_partitions = 31;
+    ListViews list_views = 32;
+    GetTableProperties get_table_properties = 33;
+    GetCreateTableString get_create_table_string = 34;
+    TruncateTable truncate_table = 35;
+    AnalyzeTable analyze_table = 36;
   }
 }
 
@@ -241,3 +251,74 @@ message ListCatalogs {
   // (Optional) The pattern that the catalog name needs to match
   optional string pattern = 1;
 }
+
+// See `spark.catalog.dropTable`
+message DropTable {
+  // (Required)
+  string table_name = 1;
+  bool if_exists = 2;
+  bool purge = 3;
+}
+
+// See `spark.catalog.dropView`
+message DropView {
+  // (Required)
+  string view_name = 1;
+  bool if_exists = 2;
+}
+
+// See `spark.catalog.createDatabase`
+message CreateDatabase {
+  // (Required)
+  string db_name = 1;
+  bool if_not_exists = 2;
+  map properties = 3;
+}
+
+// See `spark.catalog.dropDatabase`
+message DropDatabase {
+  // (Required)
+  string db_name = 1;
+  bool if_exists = 2;
+  bool cascade = 3;
+}
+
+// See `spark.catalog.listPartitions`
+message ListPartitions {
+  // (Required)
+  string table_name = 1;
+}
+
+// See `spark.catalog.listViews`
+message ListViews {
+  // (Optional)
+  optional string db_name = 1;
+  // (Optional) The pattern that the view name needs to match
+  optional string pattern = 2;
+}
+
+// See `spark.catalog.getTableProperties`
+message GetTableProperties {
+  // (Required)
+  string table_name = 1;
+}
+
+// See `spark.catalog.getCreateTableString`
+message GetCreateTableString {
+  // (Required)
+  string table_name = 1;
+  bool as_serde = 2;
+}
+
+// See `spark.catalog.truncateTable`
+message TruncateTable {
+  // (Required)
+  string table_name = 1;
+}
+
+// See `spark.catalog.analyzeTable`
+message AnalyzeTable {
+  // (Required)
+  string table_name = 1;
+  bool no_scan = 2;
+}
diff --git a/crates/sail-spark-connect/proto/spark/connect/commands.proto b/crates/sail-spark-connect/proto/spark/connect/commands.proto
index 861af17e2..dcf5aff23 100644
--- a/crates/sail-spark-connect/proto/spark/connect/commands.proto
+++ b/crates/sail-spark-connect/proto/spark/connect/commands.proto
@@ -144,6 +144,9 @@ message WriteOperation {
   // (Optional) Columns used for clustering the table.
   repeated string clustering_columns = 10;
 
+  // (Optional) Whether schema evolution is enabled for the write.
+  bool with_schema_evolution = 11;
+
   message SaveTable {
     // (Required) The table name.
     string table_name = 1;
@@ -211,6 +214,9 @@ message WriteOperationV2 {
 
   // (Optional) Columns used for clustering the table.
   repeated string clustering_columns = 9;
+
+  // (Optional) Whether schema evolution is enabled for the write.
+  bool with_schema_evolution = 10;
 }
 
 // Starts write stream operation as streaming query. Query ID and Run ID of the streaming
@@ -232,6 +238,7 @@ message WriteStreamOperationStart {
     bool available_now = 6;
     bool once = 7;
     string continuous_checkpoint_interval = 8;
+    string real_time_batch_duration = 100;
   }
 
   string output_mode = 9;
diff --git a/crates/sail-spark-connect/proto/spark/connect/pipelines.proto b/crates/sail-spark-connect/proto/spark/connect/pipelines.proto
index 0874c2d10..7632ec95b 100644
--- a/crates/sail-spark-connect/proto/spark/connect/pipelines.proto
+++ b/crates/sail-spark-connect/proto/spark/connect/pipelines.proto
@@ -22,6 +22,7 @@ package spark.connect;
 import "google/protobuf/any.proto";
 import "google/protobuf/timestamp.proto";
 import "spark/connect/common.proto";
+import "spark/connect/expressions.proto";
 import "spark/connect/relations.proto";
 import "spark/connect/types.proto";
 
@@ -40,6 +41,8 @@ message PipelineCommand {
     DefineSqlGraphElements define_sql_graph_elements = 6;
     GetQueryFunctionExecutionSignalStream get_query_function_execution_signal_stream = 7;
     DefineFlowQueryFunctionResult define_flow_query_function_result = 8;
+    ExecuteOutputFlows execute_output_flows = 9;
+
     // Reserved field for protocol extensions.
     // Used to support forward-compatibility by carrying additional command types
     // that are not yet defined in this version of the proto. During planning, the
@@ -142,6 +145,7 @@ message PipelineCommand {
 
     oneof details {
       WriteRelationFlowDetails relation_flow_details = 7;
+      AutoCdcFlowDetails auto_cdc_flow_details = 10;
       google.protobuf.Any extension = 999;
     }
 
@@ -152,6 +156,46 @@ message PipelineCommand {
       optional spark.connect.Relation relation = 1;
     }
 
+    // Details for Auto CDC flows.
+    message AutoCdcFlowDetails {
+      // The name of the CDC source to stream from.
+      optional string source = 1;
+
+      // Column(s) that uniquely identify a row in source and target data.
+      repeated Expression keys = 2;
+
+      // Expression to order the source data.
+      optional Expression sequence_by = 3;
+
+      // Delete condition for the merged operation.
+      optional Expression apply_as_deletes = 6;
+
+      // Truncate condition for the merged operation.
+      optional Expression apply_as_truncates = 7;
+
+      // Columns included in the output table.
+      repeated Expression column_list = 8;
+
+      // Columns excluded from the output table.
+      repeated Expression except_column_list = 9;
+
+      // SCD Type for target table.
+      SCDType stored_as_scd_type = 10;
+
+      // Subset of columns to ignore null in updates.
+      repeated Expression ignore_null_updates_column_list = 14;
+
+      // Subset of columns excluded from ignoring null in updates.
+      repeated Expression ignore_null_updates_except_column_list = 15;
+
+    }
+
+    // SCD Type for Auto CDC target tables.
+    enum SCDType {
+      SCD_TYPE_UNSPECIFIED = 0;
+      SCD_TYPE_1 = 1;
+    }
+
     // If true, define the flow as a one-time flow, such as for backfill.
     // Set to true changes the flow in two ways:
     //   - The flow is run one time by default. If the pipeline is ran with a full refresh,
@@ -165,6 +209,25 @@ message PipelineCommand {
     }
   }
 
+  // Request to execute all flows for a single output (dataset or sink) remotely.
+  message ExecuteOutputFlows {
+
+    // The output (table or materialized view or sink) definition.
+    optional DefineOutput define_output = 1;
+
+    // The flows to execute for this table.
+    repeated DefineFlow define_flows = 2;
+
+    // Whether to perform a full refresh instead of an incremental update.
+    optional bool full_refresh = 3;
+
+    // Storage location for pipeline checkpoints and metadata.
+    optional string storage = 4;
+
+    // Reserved field for protocol extensions.
+    repeated google.protobuf.Any extension = 999;
+  }
+
   // Resolves all datasets and flows and start a pipeline update. Should be called after all
   // graph elements are registered.
   message StartRun {
@@ -213,8 +276,13 @@ message PipelineCommand {
   // Request from the client to update the flow function evaluation result
   // for a previously un-analyzed flow.
   message DefineFlowQueryFunctionResult {
-    // The fully qualified name of the flow being updated.
-    optional string flow_name = 1;
+    // (Deprecated) The fully qualified name of the flow being updated.
+    //
+    // This field is deprecated since Spark 4.2+. Use flow_identifier field instead.
+    optional string flow_name = 1 [deprecated = true];
+
+    // The fully qualified identifier of the flow being updated.
+    optional ResolvedIdentifier flow_identifier = 4;
 
     // The ID of the graph this flow belongs to.
     optional string dataflow_graph_id = 2;
@@ -290,7 +358,13 @@ message SourceCodeLocation {
 // A signal from the server to the client to execute the query function for one or more flows, and
 // to register their results with the server.
 message PipelineQueryFunctionExecutionSignal {
-  repeated string flow_names = 1;
+  // (Deprecated) The name of flows that are ready to be re-evaluated.
+  //
+  // This field is deprecated since Spark 4.2+. Use flow_identifiers field instead.
+  repeated string flow_names = 1 [deprecated = true];
+
+  // The identifier of flows that are ready to be re-evaluated
+  repeated ResolvedIdentifier flow_identifiers = 2;
 }
 
 // Metadata providing context about the pipeline during Spark Connect query analysis.
@@ -299,8 +373,12 @@ message PipelineAnalysisContext {
   optional string dataflow_graph_id = 1;
   // The path of the top-level pipeline file determined at runtime during pipeline initialization.
   optional string definition_path = 2;
-  // The name of the Flow involved in this analysis
-  optional string flow_name = 3;
+  // (Deprecated) The name of the Flow involved in this analysis
+  //
+  // This field is deprecated since Spark 4.2+. Use flow_identifier field instead.
+  optional string flow_name = 3 [deprecated = true];
+  // The identifier of the Flow involved in this analysis
+  optional ResolvedIdentifier flow_identifier = 4;
 
   // Reserved field for protocol extensions.
   repeated google.protobuf.Any extension = 999;
diff --git a/crates/sail-spark-connect/proto/spark/connect/relations.proto b/crates/sail-spark-connect/proto/spark/connect/relations.proto
index 1583785e6..95cc9281d 100644
--- a/crates/sail-spark-connect/proto/spark/connect/relations.proto
+++ b/crates/sail-spark-connect/proto/spark/connect/relations.proto
@@ -81,6 +81,8 @@ message Relation {
     UnresolvedTableValuedFunction unresolved_table_valued_function = 43;
     LateralJoin lateral_join = 44;
     ChunkedCachedLocalRelation chunked_cached_local_relation = 45;
+    RelationChanges relation_changes = 46;
+    NearestByJoin nearest_by_join = 47;
 
     // NA functions
     NAFill fill_na = 90;
@@ -256,9 +258,33 @@ message Read {
     //
     // This is only supported by the JDBC data source.
     repeated string predicates = 5;
+
+    // (Optional) A user-provided name for the streaming source.
+    // This name is used in checkpoint metadata and enables stable checkpoint locations
+    // for source evolution.
+    optional string source_name = 6;
   }
 }
 
+// Reads Change Data Capture (CDC) changes for a named table.
+//
+// This corresponds to the `DataFrameReader.changes()` or `DataStreamReader.changes()` API.
+// CDC-specific options (startingVersion, endingVersion, startingTimestamp, endingTimestamp,
+// deduplicationMode, computeUpdates, etc.) are passed in the options map.
+message RelationChanges {
+  // (Required) Unparsed identifier for the table.
+  string unparsed_identifier = 1;
+
+  // Options for the CDC query. The map key is case insensitive.
+  // Supported keys include: startingVersion, endingVersion, startingTimestamp,
+  // endingTimestamp, deduplicationMode, computeUpdates, startingBoundInclusive,
+  // endingBoundInclusive.
+  map options = 2;
+
+  // (Optional) Indicates if this is a streaming CDC read.
+  bool is_streaming = 3;
+}
+
 // Projection of a bag of expressions for a given input relation.
 //
 // The input relation must be specified.
@@ -1184,12 +1210,13 @@ message Parse {
   // (Optional) DataType representing the schema. If not set, Spark will infer the schema.
   optional DataType schema = 3;
 
-  // Options for the csv/json parser. The map key is case insensitive.
+  // Options for the csv/json/xml parser. The map key is case insensitive.
   map options = 4;
   enum ParseFormat {
     PARSE_FORMAT_UNSPECIFIED = 0;
     PARSE_FORMAT_CSV = 1;
     PARSE_FORMAT_JSON = 2;
+    PARSE_FORMAT_XML = 3;
   }
 }
 
@@ -1250,3 +1277,33 @@ message LateralJoin {
   // (Required) The join type.
   Join.JoinType join_type = 4;
 }
+
+// Relation of type [[NearestByJoin]].
+//
+// For each row on the left side, returns up to `num_results` rows from the right side ranked
+// by `ranking_expression`.
+message NearestByJoin {
+  // (Required) Left (query) input relation.
+  Relation left = 1;
+
+  // (Required) Right (base) input relation.
+  Relation right = 2;
+
+  // (Required) Scalar expression used to rank candidate rows on the right side.
+  Expression ranking_expression = 3;
+
+  // (Required) Maximum number of matches per left row. Must be between 1 and 100000.
+  int32 num_results = 4;
+
+  // The following three fields use `string` (not typed enums) for parity with `AsOfJoin`,
+  // which models analogous fields the same way. Validation happens server-side at planning time.
+
+  // (Required) The join type. Must be one of: "inner", "leftouter".
+  string join_type = 5;
+
+  // (Required) Search algorithm contract. Must be one of: "approx", "exact".
+  string mode = 6;
+
+  // (Required) Ranking direction. Must be one of: "distance", "similarity".
+  string direction = 7;
+}
diff --git a/crates/sail-spark-connect/src/config.rs b/crates/sail-spark-connect/src/config.rs
index 088cd610c..a066b2867 100644
--- a/crates/sail-spark-connect/src/config.rs
+++ b/crates/sail-spark-connect/src/config.rs
@@ -1,21 +1,12 @@
 use std::collections::HashMap;
-use std::sync::{Arc, OnceLock};
+use std::sync::Arc;
 
 use sail_plan::config::{DefaultTimestampType, PlanConfig};
 use sail_python_udf::config::PySparkUdfConfig;
 
 use crate::error::{SparkError, SparkResult};
 use crate::spark::config::{
-    SPARK_CONFIG, SPARK_SQL_ANSI_ENABLED, SPARK_SQL_CASE_SENSITIVE, SPARK_SQL_CROSS_JOIN_ENABLED,
-    SPARK_SQL_EXECUTION_ARROW_MAX_RECORDS_PER_BATCH, SPARK_SQL_EXECUTION_ARROW_USE_LARGE_VAR_TYPES,
-    SPARK_SQL_EXECUTION_PANDAS_CONVERT_TO_ARROW_ARRAY_SAFELY,
-    SPARK_SQL_EXECUTION_PYSPARK_BINARY_AS_BYTES,
-    SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_INT_TO_DECIMAL_COERCION_ENABLED,
-    SPARK_SQL_LEGACY_EXECUTION_PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME,
-    SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDF_PANDAS_CONVERSION_ENABLED,
-    SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDTF_PANDAS_CONVERSION_ENABLED, SPARK_SQL_PIVOT_MAX_VALUES,
-    SPARK_SQL_SESSION_TIME_ZONE, SPARK_SQL_SOURCES_DEFAULT, SPARK_SQL_TIMESTAMP_TYPE,
-    SPARK_SQL_WAREHOUSE_DIR,
+    SPARK_CONFIG_V3_5, SPARK_CONFIG_V4_0, SPARK_CONFIG_V4_1, SPARK_CONFIG_V4_2, SparkConfigKey,
 };
 use crate::spark::connect;
 
@@ -44,23 +35,50 @@ impl From for connect::KeyValue {
 }
 
 pub(crate) struct SparkRuntimeConfig {
+    entries:
+        &'static phf::Map<&'static str, &'static crate::spark::config::SparkConfigEntry<'static>>,
     config: HashMap,
 }
 
 impl SparkRuntimeConfig {
-    pub(crate) fn new() -> Self {
-        Self {
+    pub(crate) fn try_new() -> SparkResult {
+        let entries = match get_pyspark_version() {
+            Ok(version) => {
+                let mut parts = version.split('.');
+                match (parts.next(), parts.next()) {
+                    // Use the Spark 3.5 configuration to provide best-effort support
+                    // for all 3.x versions.
+                    (Some("3"), _) => &SPARK_CONFIG_V3_5,
+                    (Some("4"), Some("0")) => &SPARK_CONFIG_V4_0,
+                    (Some("4"), Some("1")) => &SPARK_CONFIG_V4_1,
+                    (Some("4"), Some("2")) => &SPARK_CONFIG_V4_2,
+                    _ => {
+                        return Err(SparkError::invalid(format!(
+                            "unsupported PySpark version: {version}"
+                        )));
+                    }
+                }
+            }
+            Err(_) => {
+                // Use the earliest Spark configuration when we cannot determine the PySpark version,
+                // which can happen when running Rust tests for example.
+                &SPARK_CONFIG_V3_5
+            }
+        };
+        Ok(Self {
+            entries,
             config: HashMap::new(),
-        }
+        })
     }
 
-    fn validate_removed_key(key: &str, value: &str) -> SparkResult<()> {
-        if let Some(entry) = SPARK_CONFIG.get(key) {
-            if entry.removed.is_some() && entry.default_value != Some(value) {
-                return Err(SparkError::invalid(format!(
-                    "configuration has been removed: {key}"
-                )));
-            }
+    fn validate_removed_key(&self, key: &str, value: &str) -> SparkResult<()> {
+        if let Some(entry) = self.entries.get(key)
+            && entry.removed.is_some()
+            && entry.default_value != Some(value)
+        {
+            return Err(SparkError::invalid(format!(
+                "configuration has been removed: {key}"
+            )));
         }
         Ok(())
     }
@@ -71,7 +89,7 @@ impl SparkRuntimeConfig {
         if let Some(value) = self.config.get(key) {
             return Some(value.as_str());
         }
-        let entry = SPARK_CONFIG.get(key);
+        let entry = self.entries.get(key);
         for alt in entry.map(|x| x.alternatives).unwrap_or(&[]) {
             if let Some(value) = self.config.get(*alt) {
                 return Some(value.as_str());
@@ -84,13 +102,10 @@ impl SparkRuntimeConfig {
         if let Some(value) = self.get_by_key(key) {
             return Ok(Some(value));
         }
-        let entry = SPARK_CONFIG.get(key);
+        let entry = self.entries.get(key);
         if let Some(fallback) = entry.and_then(|x| x.fallback) {
             return self.get(fallback);
         }
-        if let Some(value) = versioned_default_value(key) {
-            return Ok(Some(value));
-        }
         if let Some(entry) = entry {
             return Ok(entry.default_value);
         }
@@ -103,13 +118,10 @@ impl SparkRuntimeConfig {
         if let Some(value) = self.get_by_key(key) {
             return Some(value);
         }
-        let entry = SPARK_CONFIG.get(key);
+        let entry = self.entries.get(key);
         if let Some(fallback) = entry.and_then(|x| x.fallback) {
             return self.get_option(fallback);
         }
-        if let Some(value) = versioned_default_value(key) {
-            return Some(value);
-        }
         entry.and_then(|x| x.default_value)
     }
 
@@ -121,18 +133,15 @@ impl SparkRuntimeConfig {
         if let Some(value) = self.get_by_key(key) {
             return Some(value);
         }
-        let entry = SPARK_CONFIG.get(key);
+        let entry = self.entries.get(key);
         if let Some(fallback) = entry.and_then(|x| x.fallback) {
             return self.get_with_default(fallback, default);
         }
-        if let Some(value) = versioned_default_value(key) {
-            return Some(value);
-        }
         default
     }
 
     pub(crate) fn set(&mut self, key: String, value: String) -> SparkResult<()> {
-        Self::validate_removed_key(key.as_str(), value.as_str())?;
+        self.validate_removed_key(key.as_str(), value.as_str())?;
         self.config.insert(key, value);
         Ok(())
     }
@@ -159,58 +168,39 @@ impl SparkRuntimeConfig {
             .collect())
     }
 
-    pub(crate) fn is_modifiable(key: &str) -> bool {
-        SPARK_CONFIG
+    pub(crate) fn is_modifiable(&self, key: &str) -> bool {
+        self.entries
             .get(key)
             .map(|entry| !entry.is_static && entry.removed.is_none())
             .unwrap_or(false)
     }
 
-    fn get_warning(key: &str) -> Option<&str> {
-        SPARK_CONFIG
+    fn get_warning(&self, key: &str) -> Option<&str> {
+        self.entries
             .get(key)
             .and_then(|entry| entry.deprecated.as_ref())
             .map(|x| x.comment)
     }
 
-    pub(crate) fn get_warnings(kv: &[ConfigKeyValue]) -> Vec {
+    pub(crate) fn get_warnings(&self, kv: &[ConfigKeyValue]) -> Vec {
         kv.iter()
-            .flat_map(|x| Self::get_warning(x.key.as_str()))
+            .flat_map(|x| self.get_warning(x.key.as_str()))
             .map(|x| x.to_string())
             .collect()
     }
 
-    pub(crate) fn get_warnings_by_keys(keys: &[String]) -> Vec {
+    pub(crate) fn get_warnings_by_keys(&self, keys: &[String]) -> Vec {
         keys.iter()
-            .flat_map(|x| Self::get_warning(x.as_str()))
+            .flat_map(|x| self.get_warning(x.as_str()))
             .map(|x| x.to_string())
             .collect()
     }
 }
 
-fn versioned_default_value(key: &str) -> Option<&'static str> {
-    match key {
-        SPARK_SQL_ANSI_ENABLED if get_pyspark_major_version().is_some_and(|x| x < 4) => {
-            Some("false")
-        }
-        _ => None,
-    }
-}
-
-fn get_pyspark_major_version() -> Option {
-    static PYSPARK_MAJOR_VERSION: OnceLock> = OnceLock::new();
-
-    *PYSPARK_MAJOR_VERSION.get_or_init(|| {
-        get_pyspark_version()
-            .ok()
-            .and_then(|version| version.split('.').next()?.parse().ok())
-    })
-}
-
 pub(crate) fn get_pyspark_version() -> SparkResult {
+    use pyo3::Python;
     use pyo3::prelude::PyAnyMethods;
     use pyo3::types::PyModule;
-    use pyo3::Python;
 
     Python::attach(|py| {
         let module = PyModule::import(py, "pyspark")?;
@@ -220,6 +210,9 @@ pub(crate) fn get_pyspark_version() -> SparkResult {
     .map_err(|e: pyo3::PyErr| SparkError::invalid(format!("failed to get PySpark version: {e}")))
 }
 
+// We must use `get_option` when extracting values from `SparkRuntimeConfig`
+// since not all configuration keys are supported in all versions of Spark.
+
 impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
     type Error = SparkError;
 
@@ -227,14 +220,14 @@ impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
         let mut output = PlanConfig::new()?;
 
         if let Some(value) = config
-            .get(SPARK_SQL_SESSION_TIME_ZONE)?
+            .get_option(SparkConfigKey::SPARK_SQL_SESSION_TIME_ZONE)
             .map(|x| x.to_string())
         {
             output.session_timezone = Arc::from(value);
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_EXECUTION_ARROW_USE_LARGE_VAR_TYPES)?
+            .get_option(SparkConfigKey::SPARK_SQL_EXECUTION_ARROW_USE_LARGE_VAR_TYPES)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -242,17 +235,17 @@ impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_SOURCES_DEFAULT)?
+            .get_option(SparkConfigKey::SPARK_SQL_SOURCES_DEFAULT)
             .map(|x| x.to_string())
         {
             output.default_table_file_format = value;
         }
 
-        if let Some(value) = config.get(SPARK_SQL_WAREHOUSE_DIR)? {
+        if let Some(value) = config.get_option(SparkConfigKey::SPARK_SQL_WAREHOUSE_DIR) {
             output.default_warehouse_directory = value.to_string();
         }
 
-        if let Some(value) = config.get(SPARK_SQL_TIMESTAMP_TYPE)? {
+        if let Some(value) = config.get_option(SparkConfigKey::SPARK_SQL_TIMESTAMP_TYPE) {
             let value = value.to_uppercase().trim().to_string();
             if value == "TIMESTAMP_NTZ" {
                 output.default_timestamp_type = DefaultTimestampType::TimestampNtz;
@@ -266,7 +259,7 @@ impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_ANSI_ENABLED)?
+            .get_option(SparkConfigKey::SPARK_SQL_ANSI_ENABLED)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -274,7 +267,7 @@ impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_CROSS_JOIN_ENABLED)?
+            .get_option(SparkConfigKey::SPARK_SQL_CROSS_JOIN_ENABLED)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -282,7 +275,7 @@ impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_CASE_SENSITIVE)?
+            .get_option(SparkConfigKey::SPARK_SQL_CASE_SENSITIVE)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -290,13 +283,21 @@ impl TryFrom<&SparkRuntimeConfig> for PlanConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_PIVOT_MAX_VALUES)?
+            .get_option(SparkConfigKey::SPARK_SQL_PIVOT_MAX_VALUES)
             .map(|x| x.trim().parse::())
             .transpose()?
         {
             output.pivot_max_values = value;
         }
 
+        if let Some(value) = config
+            .get_option(SparkConfigKey::SPARK_SQL_TVF_ALLOW_MULTIPLE_TABLE_ARGUMENTS_ENABLED)
+            .map(|x| x.to_lowercase().parse::())
+            .transpose()?
+        {
+            output.tvf_allow_multiple_table_arguments = value;
+        }
+
         output.pyspark_udf_config = Arc::new(PySparkUdfConfig::try_from(config)?);
 
         Ok(output)
@@ -310,14 +311,14 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         let mut output = PySparkUdfConfig::default();
 
         if let Some(value) = config
-            .get(SPARK_SQL_SESSION_TIME_ZONE)?
+            .get_option(SparkConfigKey::SPARK_SQL_SESSION_TIME_ZONE)
             .map(|x| x.to_string())
         {
             output.session_timezone = value;
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_LEGACY_EXECUTION_PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME)?
+            .get_option(SparkConfigKey::SPARK_SQL_LEGACY_EXECUTION_PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -325,7 +326,7 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_EXECUTION_PANDAS_CONVERT_TO_ARROW_ARRAY_SAFELY)?
+            .get_option(SparkConfigKey::SPARK_SQL_EXECUTION_PANDAS_CONVERT_TO_ARROW_ARRAY_SAFELY)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -333,7 +334,7 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_EXECUTION_ARROW_MAX_RECORDS_PER_BATCH)?
+            .get_option(SparkConfigKey::SPARK_SQL_EXECUTION_ARROW_MAX_RECORDS_PER_BATCH)
             .map(|x| x.parse::())
             .transpose()?
         {
@@ -345,7 +346,17 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDF_PANDAS_CONVERSION_ENABLED)?
+            .get_option(SparkConfigKey::SPARK_SQL_EXECUTION_ARROW_USE_LARGE_VAR_TYPES)
+            .map(|x| x.to_lowercase().parse::())
+            .transpose()?
+        {
+            output.arrow_use_large_var_types = value;
+        }
+
+        if let Some(value) = config
+            .get_option(
+                SparkConfigKey::SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDF_PANDAS_CONVERSION_ENABLED,
+            )
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -353,7 +364,9 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDTF_PANDAS_CONVERSION_ENABLED)?
+            .get_option(
+                SparkConfigKey::SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDTF_PANDAS_CONVERSION_ENABLED,
+            )
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -361,7 +374,7 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_INT_TO_DECIMAL_COERCION_ENABLED)?
+            .get_option(SparkConfigKey::SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_INT_TO_DECIMAL_COERCION_ENABLED)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
@@ -369,7 +382,17 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig {
         }
 
         if let Some(value) = config
-            .get(SPARK_SQL_EXECUTION_PYSPARK_BINARY_AS_BYTES)?
+            .get_option(
+                SparkConfigKey::SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_PREFER_INT_EXTENSION_DTYPE,
+            )
+            .map(|x| x.to_lowercase().parse::())
+            .transpose()?
+        {
+            output.python_udf_pandas_prefer_int_extension_dtype = value;
+        }
+
+        if let Some(value) = config
+            .get_option(SparkConfigKey::SPARK_SQL_EXECUTION_PYSPARK_BINARY_AS_BYTES)
             .map(|x| x.to_lowercase().parse::())
             .transpose()?
         {
diff --git a/crates/sail-spark-connect/src/debug.rs b/crates/sail-spark-connect/src/debug.rs
index f9e677904..a505d061a 100644
--- a/crates/sail-spark-connect/src/debug.rs
+++ b/crates/sail-spark-connect/src/debug.rs
@@ -1,9 +1,9 @@
 use std::fmt::{Debug, Formatter};
 
-use sail_common::debug::DebugBinary;
+use sail_common::utils::debug::DebugBinary;
 
-use crate::spark::connect::execute_plan_response::ArrowBatch;
 use crate::spark::connect::LocalRelation;
+use crate::spark::connect::execute_plan_response::ArrowBatch;
 
 impl Debug for ArrowBatch {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
diff --git a/crates/sail-spark-connect/src/entrypoint.rs b/crates/sail-spark-connect/src/entrypoint.rs
index 20dc2a3a8..1a7eaf038 100644
--- a/crates/sail-spark-connect/src/entrypoint.rs
+++ b/crates/sail-spark-connect/src/entrypoint.rs
@@ -1,9 +1,10 @@
 use std::future::Future;
 use std::sync::Arc;
 
+use sail_common::actor::ActorSystem;
 use sail_common::config::{AppConfig, GRPC_MAX_MESSAGE_LENGTH_DEFAULT};
 use sail_common::runtime::RuntimeHandle;
-use sail_server::ServerBuilder;
+use sail_common::server::ServerBuilder;
 pub use sail_session::session_manager::SessionManagerOptions;
 use tokio::net::TcpListener;
 use tonic::codec::CompressionEncoding;
@@ -22,19 +23,26 @@ pub async fn serve(
 where
     F: Future,
 {
-    let session_manager = create_spark_session_manager(config, runtime)?;
-    let server = SparkConnectServer::new(session_manager);
-    let service = SparkConnectServiceServer::new(server)
-        // The original Spark Connect server seems to have configuration for inbound (decoding) message size only.
-        // .max_encoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
-        .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
-        .accept_compressed(CompressionEncoding::Gzip)
-        .accept_compressed(CompressionEncoding::Zstd)
-        .send_compressed(CompressionEncoding::Gzip)
-        .send_compressed(CompressionEncoding::Zstd);
-    ServerBuilder::new("sail_spark_connect", Default::default())
-        .add_service(service, Some(crate::spark::connect::FILE_DESCRIPTOR_SET))
-        .await
-        .serve(listener, signal)
-        .await
+    let mut system = ActorSystem::new();
+    let session_manager = create_spark_session_manager(config, runtime, &mut system).await?;
+    let result = {
+        let server = SparkConnectServer::new(session_manager.clone());
+        let service = SparkConnectServiceServer::new(server)
+            // The original Spark Connect server seems to have configuration for inbound (decoding) message size only.
+            // .max_encoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
+            .max_decoding_message_size(GRPC_MAX_MESSAGE_LENGTH_DEFAULT)
+            .accept_compressed(CompressionEncoding::Gzip)
+            .accept_compressed(CompressionEncoding::Zstd)
+            .send_compressed(CompressionEncoding::Gzip)
+            .send_compressed(CompressionEncoding::Zstd);
+        ServerBuilder::new("sail_spark_connect", Default::default())
+            .add_service(service, Some(crate::spark::connect::FILE_DESCRIPTOR_SET))
+            .await
+            .serve(listener, signal)
+            .await
+            .map_err(|e| std::io::Error::other(e.to_string()))
+    };
+    session_manager.shutdown().await?;
+    system.join().await;
+    result.map_err(Into::into)
 }
diff --git a/crates/sail-spark-connect/src/error.rs b/crates/sail-spark-connect/src/error.rs
index c4f4208ee..4597f81f6 100644
--- a/crates/sail-spark-connect/src/error.rs
+++ b/crates/sail-spark-connect/src/error.rs
@@ -46,6 +46,8 @@ pub enum SparkError {
     InternalError(String),
     #[error("analysis error: {0}")]
     AnalysisError(String),
+    #[error("parse error: {0}")]
+    ParseError(String),
 }
 
 impl SparkError {
@@ -217,12 +219,10 @@ pub(crate) enum SparkThrowable {
     #[expect(dead_code)]
     StreamingQueryException(String),
     QueryExecutionException(String),
-    #[expect(dead_code)]
     NumberFormatException(String),
     IllegalArgumentException(String),
     ArithmeticException(String),
     UnsupportedOperationException(String),
-    #[expect(dead_code)]
     ArrayIndexOutOfBoundsException(String),
     DateTimeException(String),
     SparkRuntimeException(String),
@@ -310,15 +310,8 @@ fn truncate_grpc_message(message: &str) -> String {
     let head_take = keep / 3;
     let tail_take = keep - head_take;
 
-    let mut head_end = head_take;
-    while head_end > 0 && !message.is_char_boundary(head_end) {
-        head_end -= 1;
-    }
-
-    let mut tail_start = message.len().saturating_sub(tail_take);
-    while tail_start < message.len() && !message.is_char_boundary(tail_start) {
-        tail_start += 1;
-    }
+    let head_end = message.floor_char_boundary(head_take);
+    let tail_start = message.ceil_char_boundary(message.len().saturating_sub(tail_take));
 
     format!(
         "{}{TRUNCATED_MARKER}{}",
@@ -393,13 +386,15 @@ impl From for SparkThrowable {
                     SparkThrowable::PythonException(message)
                 }
             }
-            CommonErrorCause::ArrowCast(x)
-            | CommonErrorCause::Schema(x)
+            CommonErrorCause::ArrowCast(x) => cast_error_to_throwable(x),
+            CommonErrorCause::Schema(x)
             | CommonErrorCause::Plan(x)
             | CommonErrorCause::Configuration(x) => SparkThrowable::AnalysisException(x),
             CommonErrorCause::Execution(x) => {
                 if is_timestamp_parse_error(&x) {
                     SparkThrowable::DateTimeException(x)
+                } else if is_array_index_out_of_bounds_error(&x) {
+                    SparkThrowable::ArrayIndexOutOfBoundsException(x)
                 } else {
                     // TODO: handle situations where a different exception type is more appropriate.
                     SparkThrowable::AnalysisException(x)
@@ -414,6 +409,29 @@ fn is_timestamp_parse_error(message: &str) -> bool {
     message.starts_with("Error parsing timestamp")
 }
 
+fn is_array_index_out_of_bounds_error(message: &str) -> bool {
+    message.contains("[INVALID_ARRAY_INDEX]")
+}
+
+fn cast_error_to_throwable(message: String) -> SparkThrowable {
+    // Spark ANSI cast failures map by target type:
+    //   numeric target -> NumberFormatException   (CAST('abc' AS DOUBLE/INT/...))
+    //   boolean target -> SparkRuntimeException    (CAST('abc' AS BOOLEAN))
+    // Arrow's `CastError` message ends with "to value of  type".
+    if message.contains("Boolean type") {
+        SparkThrowable::SparkRuntimeException(message)
+    } else if [
+        "Int8", "Int16", "Int32", "Int64", "UInt", "Float16", "Float32", "Float64", "Decimal",
+    ]
+    .iter()
+    .any(|t| message.contains(t))
+    {
+        SparkThrowable::NumberFormatException(message)
+    } else {
+        SparkThrowable::AnalysisException(message)
+    }
+}
+
 impl From for Status {
     fn from(error: SparkError) -> Self {
         match error {
@@ -434,6 +452,7 @@ impl From for Status {
                 SparkThrowable::UnsupportedOperationException(s).into()
             }
             SparkError::AnalysisError(s) => SparkThrowable::AnalysisException(s).into(),
+            SparkError::ParseError(s) => SparkThrowable::ParseException(s).into(),
             e @ SparkError::SendError(_) => {
                 Status::cancelled(truncate_grpc_message(&e.to_string()))
             }
diff --git a/crates/sail-spark-connect/src/executor.rs b/crates/sail-spark-connect/src/executor.rs
index 69c248f80..0ebc2ad21 100644
--- a/crates/sail-spark-connect/src/executor.rs
+++ b/crates/sail-spark-connect/src/executor.rs
@@ -7,12 +7,13 @@ use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use datafusion::arrow::array::RecordBatch;
+use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::arrow::ipc::writer::StreamWriter;
 use datafusion::execution::SendableRecordBatchStream;
-use fastrace::future::FutureExt;
 use fastrace::Span;
-use futures::stream::{StreamExt, TryStreamExt};
+use fastrace::future::FutureExt;
 use futures::Stream;
+use futures::stream::StreamExt;
 use tokio::sync::{mpsc, oneshot};
 use tokio::task::JoinHandle;
 use tonic::codegen::tokio_stream::wrappers::ReceiverStream;
@@ -28,6 +29,7 @@ use crate::spark::connect::{
 
 #[derive(Clone, Debug)]
 pub enum ExecutorBatch {
+    Heartbeat,
     ArrowBatch(ArrowBatch),
     SqlCommandResult(Box),
     WriteStreamOperationStartResult(Box),
@@ -57,23 +59,30 @@ impl ExecutorOutput {
     }
 }
 
-pub type ExecutorOutputStream = Pin + Send>>;
+pub type ExecutorOutputStream = Pin> + Send>>;
 
 struct ExecutorBuffer {
+    capacity: usize,
     inner: VecDeque,
 }
 
+// TODO: use "spark.connect.execute.reattachable.observerRetryBufferSize"
+// TODO: limit the size based on serialized message size instead of element count
+const EXECUTOR_BUFFER_CAPACITY: usize = 128;
+
 impl ExecutorBuffer {
-    fn new() -> Self {
-        // TODO: use "spark.connect.execute.reattachable.observerRetryBufferSize"
-        // TODO: limit the size based on serialized message size instead of element count
+    fn new(capacity: usize) -> Self {
         Self {
-            inner: VecDeque::with_capacity(128),
+            capacity,
+            inner: VecDeque::with_capacity(capacity),
         }
     }
 
     fn add(&mut self, output: ExecutorOutput) {
-        if self.inner.len() >= self.inner.capacity() {
+        if self.capacity == 0 {
+            return;
+        }
+        if self.inner.len() >= self.capacity {
             self.inner.pop_front();
         }
         self.inner.push_back(output);
@@ -125,38 +134,76 @@ struct ExecutorTask {
 struct ExecutorTaskContext {
     stream: SendableRecordBatchStream,
     heartbeat_interval: Duration,
+    state: ExecutorTaskState,
     buffer: Arc>,
 }
 
-impl ExecutorTaskContext {
-    fn new(stream: SendableRecordBatchStream, heartbeat_interval: Duration) -> Self {
-        Self {
-            stream,
-            heartbeat_interval,
-            buffer: Arc::new(Mutex::new(ExecutorBuffer::new())),
+pub(crate) enum ExecutorMode {
+    Query,
+    Command {
+        completion: CommandCompletionHandler,
+    },
+}
+
+impl ExecutorMode {
+    pub(crate) fn command() -> Self {
+        Self::command_with_completion(|_, _| Ok(None))
+    }
+
+    pub(crate) fn command_with_completion(
+        completion: impl FnOnce(SchemaRef, Vec) -> SparkResult>
+        + Send
+        + 'static,
+    ) -> Self {
+        Self::Command {
+            completion: Box::new(completion),
         }
     }
+}
+
+pub(crate) enum ExecutorTaskState {
+    Query,
+    Command {
+        batches: Vec,
+        completion: Option,
+    },
+}
 
-    async fn next(&mut self) -> SparkResult> {
+enum ExecutorTaskItem {
+    Batch(Option),
+    Heartbeat,
+}
+
+pub(crate) type CommandCompletionHandler =
+    Box) -> SparkResult> + Send>;
+
+impl ExecutorTaskContext {
+    async fn next(
+        stream: &mut SendableRecordBatchStream,
+        heartbeat_interval: Duration,
+    ) -> SparkResult {
         let span = Span::enter_with_local_parent("ExecutorTaskContext::next");
         tokio::select! {
-            batch = self.stream.next().in_span(span) => Ok(batch.transpose()?),
-            _ = tokio::time::sleep(self.heartbeat_interval) => {
-                Ok(Some(RecordBatch::new_empty(self.stream.schema())))
+            batch = stream.next().in_span(span) => Ok(ExecutorTaskItem::Batch(batch.transpose()?)),
+            _ = tokio::time::sleep(heartbeat_interval) => {
+                // FIXME: non-reattachable clients cannot refresh session activity by releasing heartbeat responses
+                Ok(ExecutorTaskItem::Heartbeat)
             }
         }
     }
 
-    fn save_output(&self, output: &ExecutorOutput) -> SparkResult<()> {
-        let mut buffer = self.buffer.lock()?;
-        buffer.add(output.clone());
+    async fn send_output(
+        buffer: &Arc>,
+        tx: &mpsc::Sender>,
+        output: ExecutorOutput,
+    ) -> SparkResult<()> {
+        {
+            let mut buffer = buffer.lock()?;
+            buffer.add(output.clone());
+        }
+        tx.send(Ok(output)).await?;
         Ok(())
     }
-
-    fn replay_outputs(&self) -> SparkResult> {
-        let buffer = self.buffer.lock()?;
-        Ok(buffer.iter().cloned().collect())
-    }
 }
 
 enum ExecutorTaskResult {
@@ -170,11 +217,29 @@ impl Executor {
         metadata: ExecutorMetadata,
         stream: SendableRecordBatchStream,
         heartbeat_interval: Duration,
+        mode: ExecutorMode,
     ) -> Self {
+        let state = match mode {
+            ExecutorMode::Query => ExecutorTaskState::Query,
+            ExecutorMode::Command { completion } => ExecutorTaskState::Command {
+                batches: vec![],
+                completion: Some(completion),
+            },
+        };
+        let buffer = if metadata.reattachable {
+            ExecutorBuffer::new(EXECUTOR_BUFFER_CAPACITY)
+        } else {
+            ExecutorBuffer::new(0)
+        };
         Self {
             metadata,
             state: Mutex::new(ExecutorState::Pending {
-                context: ExecutorTaskContext::new(stream, heartbeat_interval),
+                context: ExecutorTaskContext {
+                    stream,
+                    heartbeat_interval,
+                    state,
+                    buffer: Arc::new(Mutex::new(buffer)),
+                },
                 span: Span::enter_with_local_parent("Executor::new"),
             }),
         }
@@ -182,55 +247,139 @@ impl Executor {
 
     async fn run_internal(
         context: &mut ExecutorTaskContext,
-        tx: mpsc::Sender,
+        tx: &mpsc::Sender>,
+        reattachable: bool,
     ) -> SparkResult<()> {
-        for out in context.replay_outputs()? {
-            tx.send(out).await?;
-        }
-        let schema = to_spark_schema(context.stream.schema())?;
-        let out = ExecutorOutput::new(ExecutorBatch::Schema(Box::new(schema)));
-        context.save_output(&out)?;
-        tx.send(out).await?;
-
-        let mut empty = true;
-        while let Some(batch) = context.next().await? {
-            let batch = to_arrow_batch(&batch)?;
-            let out = ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch));
-            context.save_output(&out)?;
-            tx.send(out).await?;
-            empty = false;
+        let outputs = {
+            let buffer = context.buffer.lock()?;
+            buffer.iter().cloned().collect::>()
+        };
+        for output in outputs {
+            tx.send(Ok(output)).await?;
         }
-        if empty {
-            let batch = RecordBatch::new_empty(context.stream.schema());
-            let batch = to_arrow_batch(&batch)?;
-            let out = ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch));
-            context.save_output(&out)?;
-            tx.send(out).await?;
+        match &mut context.state {
+            ExecutorTaskState::Query => {
+                let schema = to_spark_schema(context.stream.schema())?;
+                ExecutorTaskContext::send_output(
+                    &context.buffer,
+                    tx,
+                    ExecutorOutput::new(ExecutorBatch::Schema(Box::new(schema))),
+                )
+                .await?;
+
+                let mut empty = true;
+                loop {
+                    match ExecutorTaskContext::next(&mut context.stream, context.heartbeat_interval)
+                        .await?
+                    {
+                        ExecutorTaskItem::Batch(Some(batch)) => {
+                            let batch = to_arrow_batch(&batch)?;
+                            ExecutorTaskContext::send_output(
+                                &context.buffer,
+                                tx,
+                                ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch)),
+                            )
+                            .await?;
+                            empty = false;
+                        }
+                        ExecutorTaskItem::Batch(None) => break,
+                        ExecutorTaskItem::Heartbeat => {
+                            ExecutorTaskContext::send_output(
+                                &context.buffer,
+                                tx,
+                                ExecutorOutput::new(ExecutorBatch::Heartbeat),
+                            )
+                            .await?;
+                        }
+                    }
+                }
+                if empty {
+                    let batch = RecordBatch::new_empty(context.stream.schema());
+                    let batch = to_arrow_batch(&batch)?;
+                    ExecutorTaskContext::send_output(
+                        &context.buffer,
+                        tx,
+                        ExecutorOutput::new(ExecutorBatch::ArrowBatch(batch)),
+                    )
+                    .await?;
+                }
+                if reattachable {
+                    ExecutorTaskContext::send_output(
+                        &context.buffer,
+                        tx,
+                        ExecutorOutput::complete(),
+                    )
+                    .await?;
+                }
+            }
+            ExecutorTaskState::Command {
+                batches,
+                completion,
+            } => {
+                loop {
+                    match ExecutorTaskContext::next(&mut context.stream, context.heartbeat_interval)
+                        .await?
+                    {
+                        ExecutorTaskItem::Batch(Some(batch)) => {
+                            batches.push(batch);
+                        }
+                        ExecutorTaskItem::Batch(None) => break,
+                        ExecutorTaskItem::Heartbeat => {
+                            ExecutorTaskContext::send_output(
+                                &context.buffer,
+                                tx,
+                                ExecutorOutput::new(ExecutorBatch::Heartbeat),
+                            )
+                            .await?;
+                        }
+                    }
+                }
+                let schema = context.stream.schema();
+                let output = completion
+                    .take()
+                    .map(|completion| completion(schema, mem::take(batches)))
+                    .transpose()?
+                    .flatten();
+                if let Some(output) = output {
+                    ExecutorTaskContext::send_output(&context.buffer, tx, output).await?;
+                }
+                if reattachable {
+                    ExecutorTaskContext::send_output(
+                        &context.buffer,
+                        tx,
+                        ExecutorOutput::complete(),
+                    )
+                    .await?;
+                }
+            }
         }
-
-        let out = ExecutorOutput::new(ExecutorBatch::Complete);
-        context.save_output(&out)?;
-        tx.send(out).await?;
         Ok(())
     }
 
     async fn run(
         mut context: ExecutorTaskContext,
         listener: oneshot::Receiver<()>,
-        tx: mpsc::Sender,
+        tx: mpsc::Sender>,
+        reattachable: bool,
     ) -> ExecutorTaskResult {
         let out = tokio::select! {
-            x = Executor::run_internal(&mut context, tx) => x,
+            x = Executor::run_internal(&mut context, &tx, reattachable) => x,
             _ = listener => return ExecutorTaskResult::Paused(context),
         };
         match out {
             Ok(()) => ExecutorTaskResult::Completed,
             Err(SparkError::SendError(_)) => ExecutorTaskResult::Paused(context),
-            Err(e) => ExecutorTaskResult::Failed(e),
+            Err(e) => {
+                let _ = tx.send(Err(e)).await;
+                // TODO: track the original error in the task result
+                ExecutorTaskResult::Failed(SparkError::internal(
+                    "task failed while executing the plan",
+                ))
+            }
         }
     }
 
-    pub(crate) fn start(&self) -> SparkResult> {
+    pub(crate) fn start(&self) -> SparkResult {
         let mut state = self.state.lock()?;
         let (context, span) = match mem::replace(state.deref_mut(), ExecutorState::Idle) {
             ExecutorState::Pending { context, span } => (context, span),
@@ -256,9 +405,14 @@ impl Executor {
         let (tx, rx) = mpsc::channel(1);
         let (notifier, listener) = oneshot::channel();
         let buffer = Arc::clone(&context.buffer);
+        let reattachable = self.metadata.reattachable;
         let handle = {
             let span = { Span::enter_with_parent("Executor::run", &span) };
-            tokio::spawn(async move { Executor::run(context, listener, tx).in_span(span).await })
+            tokio::spawn(async move {
+                Executor::run(context, listener, tx, reattachable)
+                    .in_span(span)
+                    .await
+            })
         };
         *state = ExecutorState::Running {
             task: ExecutorTask {
@@ -268,7 +422,7 @@ impl Executor {
             },
             span,
         };
-        Ok(ReceiverStream::new(rx))
+        Ok(Box::pin(ReceiverStream::new(rx)))
     }
 
     pub(crate) async fn pause_if_running(&self) -> SparkResult<()> {
@@ -295,28 +449,20 @@ impl Executor {
         Ok(())
     }
 
-    pub(crate) fn release(&self, response_id: Option) -> SparkResult<()> {
+    pub(crate) fn release(&self, response_id: String) -> SparkResult<()> {
         let state = self.state.lock()?;
         let buffer = match state.deref() {
             ExecutorState::Running { task, span: _ } => &task.buffer,
             ExecutorState::Pending { context, span: _ } => &context.buffer,
             ExecutorState::Idle | ExecutorState::Failed(_) | ExecutorState::Pausing => {
-                return Ok(())
+                return Ok(());
             }
         };
-        if let Some(response_id) = response_id {
-            buffer.lock()?.remove_until(&response_id);
-        }
+        buffer.lock()?.remove_until(&response_id);
         Ok(())
     }
 }
 
-pub(crate) async fn read_stream(
-    stream: SendableRecordBatchStream,
-) -> SparkResult> {
-    stream.err_into().try_collect::>().await
-}
-
 pub(crate) fn to_arrow_batch(batch: &RecordBatch) -> SparkResult {
     let mut output = ArrowBatch::default();
     {
diff --git a/crates/sail-spark-connect/src/proto/data_type.rs b/crates/sail-spark-connect/src/proto/data_type.rs
index d31a79fb0..903eb9a4d 100644
--- a/crates/sail-spark-connect/src/proto/data_type.rs
+++ b/crates/sail-spark-connect/src/proto/data_type.rs
@@ -7,7 +7,7 @@ use sail_sql_analyzer::parser::parse_data_type;
 
 use crate::error::{ProtoFieldExt, SparkError, SparkResult};
 use crate::proto::data_type_json::parse_spark_json_data_type;
-use crate::spark::connect::{data_type as sdt, DataType};
+use crate::spark::connect::{DataType, data_type as sdt};
 
 pub(crate) const DEFAULT_FIELD_NAME: &str = "value";
 
@@ -27,20 +27,21 @@ pub(crate) const SPARK_DECIMAL_SYSTEM_DEFAULT_SCALE: i8 = 18;
 pub(crate) fn parse_spark_data_type(schema: &str) -> SparkResult {
     if let Ok(dt) = parse_data_type(schema).and_then(from_ast_data_type) {
         Ok(dt)
-    } else if let Ok(dt) =
-        parse_data_type(format!("struct<{schema}>").as_str()).and_then(from_ast_data_type)
-    {
-        match dt {
-            spec::DataType::Struct { fields } if fields.is_empty() => {
-                Err(SparkError::invalid("empty data type"))
-            }
-            // The SQL parser supports both `struct` and `struct` syntax.
-            // Therefore, by wrapping the input with `struct<...>`, we do not need separate logic
-            // to parse table schema input (`name type, ...`).
-            _ => Ok(dt),
-        }
     } else {
-        parse_spark_json_data_type(schema)?.try_into()
+        match parse_data_type(format!("struct<{schema}>").as_str()).and_then(from_ast_data_type) {
+            Ok(dt) => match dt {
+                spec::DataType::Struct { fields } if fields.is_empty() => {
+                    Err(SparkError::invalid("empty data type"))
+                }
+                // The SQL parser supports both `struct` and `struct` syntax.
+                // Therefore, by wrapping the input with `struct<...>`, we do not need separate logic
+                // to parse table schema input (`name type, ...`).
+                _ => Ok(dt),
+            },
+            Err(ddl_error) => parse_spark_json_data_type(schema)
+                .and_then(|dt| dt.try_into())
+                .map_err(|_| SparkError::ParseError(ddl_error.to_string())),
+        }
     }
 }
 
@@ -304,7 +305,7 @@ mod tests {
     use sail_common::spec;
     use sail_common::tests::test_gold_set;
 
-    use super::{parse_spark_data_type, DEFAULT_FIELD_NAME};
+    use super::{DEFAULT_FIELD_NAME, parse_spark_data_type};
     use crate::error::{SparkError, SparkResult};
 
     #[test]
diff --git a/crates/sail-spark-connect/src/proto/data_type_arrow.rs b/crates/sail-spark-connect/src/proto/data_type_arrow.rs
index 7241c70e7..2fc3c7a79 100644
--- a/crates/sail-spark-connect/src/proto/data_type_arrow.rs
+++ b/crates/sail-spark-connect/src/proto/data_type_arrow.rs
@@ -1,11 +1,10 @@
-use arrow_schema::extension::ExtensionType;
 use datafusion::arrow::datatypes as adt;
-use parquet_variant_compute::VariantType;
 use sail_common::geoarrow::extension::{GeoArrowMetadata, GeoArrowWkbType};
 use sail_common::spec;
+use sail_common_datafusion::variant::is_variant_storage_field;
 
 use crate::error::{SparkError, SparkResult};
-use crate::spark::connect::{data_type as sdt, DataType};
+use crate::spark::connect::{DataType, data_type as sdt};
 
 /// Spark geometry and geography type metadata.
 #[derive(Debug, Clone)]
@@ -106,8 +105,7 @@ impl TryFrom for sdt::StructField {
                     })),
                 },
             }
-        } else if extension_type_name == Some(VariantType::NAME) {
-            field.try_extension_type::()?;
+        } else if is_variant_storage_field(&field) {
             DataType {
                 kind: Some(sdt::Kind::Variant(sdt::Variant {
                     type_variation_reference: 0,
@@ -173,7 +171,7 @@ impl TryFrom for DataType {
             adt::DataType::Timestamp(adt::TimeUnit::Second, _)
             | adt::DataType::Timestamp(adt::TimeUnit::Millisecond, _)
             | adt::DataType::Timestamp(adt::TimeUnit::Nanosecond, _) => {
-                return Err(error(&data_type))
+                return Err(error(&data_type));
             }
             adt::DataType::Time32(adt::TimeUnit::Second) => Kind::Time(sdt::Time {
                 precision: Some(0),
@@ -341,10 +339,12 @@ mod tests {
         assert!(SparkGeoMetadata::from_json(r#"{"crs":{"type":"GeographicCRS"}}"#).is_err());
 
         // Test error on PROJJSON with unsupported authority:code
-        assert!(SparkGeoMetadata::from_json(
-            r#"{"crs":{"type":"GeographicCRS","id":{"authority":"EPSG","code":32632}}}"#
-        )
-        .is_err());
+        assert!(
+            SparkGeoMetadata::from_json(
+                r#"{"crs":{"type":"GeographicCRS","id":{"authority":"EPSG","code":32632}}}"#
+            )
+            .is_err()
+        );
 
         Ok(())
     }
diff --git a/crates/sail-spark-connect/src/proto/expression.rs b/crates/sail-spark-connect/src/proto/expression.rs
index 04a03056d..b9f65f20f 100644
--- a/crates/sail-spark-connect/src/proto/expression.rs
+++ b/crates/sail-spark-connect/src/proto/expression.rs
@@ -12,18 +12,18 @@ use sail_sql_analyzer::query::from_ast_named_expression;
 use crate::error::{ProtoFieldExt, SparkError, SparkResult};
 use crate::spark::connect::expression::cast::{CastToType, EvalMode};
 use crate::spark::connect::expression::sort_order::{NullOrdering, SortDirection};
+use crate::spark::connect::expression::window::WindowFrame;
 use crate::spark::connect::expression::window::window_frame::frame_boundary::Boundary;
 use crate::spark::connect::expression::window::window_frame::{FrameBoundary, FrameType};
-use crate::spark::connect::expression::window::WindowFrame;
 use crate::spark::connect::expression::{
     Alias, Cast, ExprType, ExpressionString, LambdaFunction, SortOrder, UnresolvedAttribute,
     UnresolvedExtractValue, UnresolvedFunction, UnresolvedNamedLambdaVariable, UnresolvedRegex,
     UnresolvedStar, UpdateFields, Window,
 };
 use crate::spark::connect::{
-    common_inline_user_defined_function as udf, common_inline_user_defined_table_function as udtf,
     CallFunction, CommonInlineUserDefinedFunction, CommonInlineUserDefinedTableFunction,
     Expression, JavaUdf, PythonUdf, PythonUdtf, ScalarScalaUdf, SubqueryExpression,
+    common_inline_user_defined_function as udf, common_inline_user_defined_table_function as udtf,
 };
 
 impl TryFrom for spec::Expr {
diff --git a/crates/sail-spark-connect/src/proto/function.rs b/crates/sail-spark-connect/src/proto/function.rs
index 0eef1bff7..8a6816d26 100644
--- a/crates/sail-spark-connect/src/proto/function.rs
+++ b/crates/sail-spark-connect/src/proto/function.rs
@@ -2,11 +2,14 @@
 mod tests {
     use std::collections::HashMap;
     use std::sync::Arc;
+    use std::thread;
 
     use datafusion::arrow::array::RecordBatch;
     use datafusion::arrow::error::ArrowError;
     use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions};
+    use futures::stream::TryStreamExt;
     use pyo3::Python;
+    use sail_common::actor::ActorSystem;
     use sail_common::config::AppConfig;
     use sail_common::runtime::RuntimeManager;
     use sail_common::tests::test_gold_set;
@@ -16,7 +19,6 @@ mod tests {
     use serde::{Deserialize, Serialize};
 
     use crate::error::{SparkError, SparkResult};
-    use crate::executor::read_stream;
     use crate::proto::data_type_json::JsonDataType;
     use crate::session::SparkSession;
     use crate::session_manager::create_spark_session_manager;
@@ -60,48 +62,60 @@ mod tests {
         let config = Arc::new(AppConfig::load()?);
         let runtime = RuntimeManager::try_new(&config.runtime)?;
         let handle = runtime.handle();
-        // We create the session manager inside an async context, even though the
-        // `SessionManager::try_new()` function itself is sync. This is because the actor system
-        // may need to spawn actors when the session runs in cluster mode.
-        let manager = handle
-            .primary()
-            .block_on(async { create_spark_session_manager(config, handle.clone()) })?;
+        let mut system = ActorSystem::new();
+        // The driver gateway is initialized before the session manager in cluster mode, so the
+        // manager must be created inside an async context.
+        let manager = handle.primary().block_on(create_spark_session_manager(
+            config,
+            handle.clone(),
+            &mut system,
+        ))?;
         let context = handle
             .primary()
             .block_on(manager.get_or_create_session_context("test".to_string(), "".to_string()))?;
-        test_gold_set(
-            "tests/gold_data/function/*.json",
-            |example: FunctionExample| -> SparkResult {
-                let relation = Relation {
-                    common: None,
-                    #[expect(deprecated)]
-                    rel_type: Some(RelType::Sql(Sql {
-                        query: example.query,
-                        args: HashMap::new(),
-                        pos_args: vec![],
-                        named_arguments: HashMap::new(),
-                        pos_arguments: vec![],
-                    })),
-                };
-                let plan = relation.try_into()?;
-                let result = handle.primary().block_on(async {
-                    let spark = context.extension::()?;
-                    let service = context.extension::()?;
-                    let (plan, _) =
-                        resolve_and_execute_plan(&context, spark.plan_config()?, plan).await?;
-                    let stream = service.runner().execute(&context, plan).await?;
-                    read_stream(stream).await
-                });
-                // TODO: validate the result against the expected output
-                // TODO: handle non-deterministic results and error messages
-                match result {
-                    // FIXME: the output can be non-deterministic
-                    Ok(_) => Ok("ok".to_string()),
-                    Err(x) => Err(x),
-                }
-            },
-            SparkError::internal,
-        )
-        .map_err(|e| e.into())
+        let test = move || {
+            test_gold_set(
+                "tests/gold_data/function/*.json",
+                |example: FunctionExample| -> SparkResult {
+                    let relation = Relation {
+                        common: None,
+                        #[expect(deprecated)]
+                        rel_type: Some(RelType::Sql(Sql {
+                            query: example.query,
+                            args: HashMap::new(),
+                            pos_args: vec![],
+                            named_arguments: HashMap::new(),
+                            pos_arguments: vec![],
+                        })),
+                    };
+                    let plan = relation.try_into()?;
+                    let result = handle.primary().block_on(async {
+                        let spark = context.extension::()?;
+                        let service = context.extension::()?;
+                        let (plan, _) =
+                            resolve_and_execute_plan(&context, spark.plan_config()?, plan).await?;
+                        let stream = service.runner().execute(&context, plan).await?;
+                        stream.err_into().try_collect::>().await
+                    });
+                    // TODO: validate the result against the expected output
+                    // TODO: handle non-deterministic results and error messages
+                    match result {
+                        // FIXME: the output can be non-deterministic
+                        Ok(_) => Ok("ok".to_string()),
+                        Err(x) => Err(x),
+                    }
+                },
+                SparkError::internal,
+            )
+        };
+        // `Handle::block_on` polls futures on the calling thread, so use a larger stack than the
+        // default to avoid stack overflow.
+        // Note that increasing the Tokio runtime stack size does not have an effect here.
+        thread::Builder::new()
+            .stack_size(8 * 1024 * 1024)
+            .spawn(test)?
+            .join()
+            .map_err(|_| SparkError::internal("test thread panicked"))?
+            .map_err(|e| e.into())
     }
 }
diff --git a/crates/sail-spark-connect/src/proto/literal.rs b/crates/sail-spark-connect/src/proto/literal.rs
index 62e52ad65..bbe7270cb 100644
--- a/crates/sail-spark-connect/src/proto/literal.rs
+++ b/crates/sail-spark-connect/src/proto/literal.rs
@@ -3,9 +3,9 @@ use sail_common::spec::data_type_to_null_literal;
 use sail_sql_analyzer::literal::numeric::parse_decimal_string;
 
 use crate::error::{ProtoFieldExt, SparkError, SparkResult};
-use crate::spark::connect::expression::literal::{Array, Decimal, LiteralType, Map, Struct, Time};
 use crate::spark::connect::expression::Literal;
-use crate::spark::connect::{data_type as sdt, DataType};
+use crate::spark::connect::expression::literal::{Array, Decimal, LiteralType, Map, Struct, Time};
+use crate::spark::connect::{DataType, data_type as sdt};
 
 impl TryFrom for spec::Literal {
     type Error = SparkError;
@@ -94,7 +94,7 @@ impl TryFrom for spec::Literal {
                         other => {
                             return Err(SparkError::invalid(format!(
                                 "Unexpected Literal type for Decimal: {other:?}"
-                            )))
+                            )));
                         }
                     }
                 }
@@ -212,7 +212,7 @@ impl TryFrom for spec::Literal {
                 }
             }
             LiteralType::SpecializedArray(_) => {
-                return Err(SparkError::todo("specialized array literal"))
+                return Err(SparkError::todo("specialized array literal"));
             }
             LiteralType::Time(Time { nano, precision }) => {
                 // Spark TIME literals carry nanoseconds since midnight.
diff --git a/crates/sail-spark-connect/src/proto/plan.rs b/crates/sail-spark-connect/src/proto/plan.rs
index d1f21395a..f7d41776c 100644
--- a/crates/sail-spark-connect/src/proto/plan.rs
+++ b/crates/sail-spark-connect/src/proto/plan.rs
@@ -2,19 +2,21 @@ use std::sync::Arc;
 
 use sail_common::spec;
 use sail_sql_analyzer::expression::{from_ast_expression, from_ast_object_name};
-use sail_sql_analyzer::parser::{parse_expression, parse_object_name, parse_one_statement};
+use sail_sql_analyzer::parser::{
+    parse_expression, parse_object_name, parse_one_statement, rewrite_positional_parameter_markers,
+};
 use sail_sql_analyzer::statement::from_ast_statement;
 
 use crate::error::{ProtoFieldExt, SparkError, SparkResult};
-use crate::proto::data_type::{parse_spark_data_type, DEFAULT_FIELD_NAME};
+use crate::proto::data_type::{DEFAULT_FIELD_NAME, parse_spark_data_type};
 use crate::spark::connect as sc;
 use crate::spark::connect::catalog::CatType;
 use crate::spark::connect::relation::RelType;
 use crate::spark::connect::write_stream_operation_start::SinkDestination;
 use crate::spark::connect::{
-    plan, Catalog, CreateDataFrameViewCommand, MergeIntoTableCommand, Plan, Relation,
-    RelationCommon, StreamingForeachFunction, TransformWithStateInfo, WriteOperation,
-    WriteOperationV2, WriteStreamOperationStart,
+    Catalog, CreateDataFrameViewCommand, MergeIntoTableCommand, Plan, Relation, RelationCommon,
+    StreamingForeachFunction, TransformWithStateInfo, WriteOperation, WriteOperationV2,
+    WriteStreamOperationStart, plan,
 };
 
 struct RelationMetadata {
@@ -88,7 +90,7 @@ impl TryFrom for spec::QueryPlan {
             plan::OpType::Root(relation) => relation,
             plan::OpType::Command(_) => return Err(SparkError::invalid("relation expected")),
             plan::OpType::CompressedOperation(_) => {
-                return Err(SparkError::unsupported("compressed operation"))
+                return Err(SparkError::unsupported("compressed operation"));
             }
         };
         relation.try_into()
@@ -204,7 +206,11 @@ impl TryFrom for RelationNode {
                             options,
                             paths,
                             predicates,
+                            source_name,
                         } = x;
+                        if source_name.is_some() {
+                            return Err(SparkError::unsupported("streaming source name"));
+                        }
                         let schema = schema
                             .and_then(|s| {
                                 if s.is_empty() {
@@ -272,7 +278,7 @@ impl TryFrom for RelationNode {
                 let join_condition = join_condition.map(|x| x.try_into()).transpose()?;
                 let join_type = match JoinType::try_from(join_type)? {
                     JoinType::Unspecified => {
-                        return Err(SparkError::invalid("unspecified join type"))
+                        return Err(SparkError::invalid("unspecified join type"));
                     }
                     JoinType::Inner => spec::JoinType::Inner,
                     JoinType::FullOuter => spec::JoinType::FullOuter,
@@ -303,7 +309,7 @@ impl TryFrom for RelationNode {
                     (Some(_), false) => {
                         return Err(SparkError::invalid(
                             "join with both condition and using columns",
-                        ))
+                        ));
                     }
                 };
                 Ok(RelationNode::Query(spec::QueryNode::Join(spec::Join {
@@ -329,7 +335,7 @@ impl TryFrom for RelationNode {
                 let right_input = right_input.required("set operation right input")?;
                 let set_op_type = match SetOpType::try_from(set_op_type)? {
                     SetOpType::Unspecified => {
-                        return Err(SparkError::invalid("unspecified set operation type"))
+                        return Err(SparkError::invalid("unspecified set operation type"));
                     }
                     SetOpType::Union => spec::SetOpType::Union,
                     SetOpType::Intersect => spec::SetOpType::Intersect,
@@ -397,7 +403,7 @@ impl TryFrom for RelationNode {
                     .collect::>>()?;
                 let node = match GroupType::try_from(group_type)? {
                     GroupType::Unspecified => {
-                        return Err(SparkError::invalid("unspecified aggregate group type"))
+                        return Err(SparkError::invalid("unspecified aggregate group type"));
                     }
                     GroupType::Groupby => {
                         if pivot.is_some() {
@@ -452,7 +458,7 @@ impl TryFrom for RelationNode {
                             .collect::>>()?;
                         spec::QueryNode::Pivot(spec::Pivot {
                             input: Box::new(input),
-                            grouping,
+                            grouping: Some(grouping),
                             aggregate,
                             columns: vec![col.try_into()?],
                             values,
@@ -491,39 +497,49 @@ impl TryFrom for RelationNode {
                     named_arguments,
                     pos_arguments,
                 } = sql;
+                let positional_arguments = match (pos_args.is_empty(), pos_arguments.is_empty()) {
+                    (false, false) => {
+                        return Err(SparkError::invalid("conflicting positional arguments"));
+                    }
+                    (false, true) => pos_args
+                        .into_iter()
+                        .map(|x| Ok(spec::Expr::Literal(x.try_into()?)))
+                        .collect::>>()?,
+                    (true, false) => pos_arguments
+                        .into_iter()
+                        .map(|x| x.try_into())
+                        .collect::>>()?,
+                    (true, true) => vec![],
+                };
+                let named_arguments = match (args.is_empty(), named_arguments.is_empty()) {
+                    (false, false) => {
+                        return Err(SparkError::invalid("conflicting named arguments"));
+                    }
+                    (false, true) => args
+                        .into_iter()
+                        .map(|(k, v)| Ok((k, spec::Expr::Literal(v.try_into()?))))
+                        .collect::>>()?,
+                    (true, false) => named_arguments
+                        .into_iter()
+                        .map(|(k, v)| Ok((k, v.try_into()?)))
+                        .collect::>>()?,
+                    (true, true) => vec![],
+                };
+                let (query, positional_marker_count) = if positional_arguments.is_empty() {
+                    (query, 0)
+                } else {
+                    rewrite_positional_parameter_markers(&query)?
+                };
+                if positional_marker_count > positional_arguments.len() {
+                    // Mirror Spark's `UNBOUND_SQL_PARAMETER`. Positional parameters are
+                    // named `_N` (0-based), so the first unbound marker is `_`.
+                    return Err(SparkError::invalid(format!(
+                        "Found the unbound parameter: _{}. Please, fix `args` and provide a mapping of the parameter to either a SQL literal or collection constructor functions such as `map()`, `array()`, `struct()`.",
+                        positional_arguments.len()
+                    )));
+                }
                 match from_ast_statement(parse_one_statement(query.as_str())?)? {
                     spec::Plan::Query(input) => {
-                        let positional_arguments =
-                            match (pos_args.is_empty(), pos_arguments.is_empty()) {
-                                (false, false) => {
-                                    return Err(SparkError::invalid(
-                                        "conflicting positional arguments",
-                                    ))
-                                }
-                                (false, true) => pos_args
-                                    .into_iter()
-                                    .map(|x| Ok(spec::Expr::Literal(x.try_into()?)))
-                                    .collect::>>()?,
-                                (true, false) => pos_arguments
-                                    .into_iter()
-                                    .map(|x| x.try_into())
-                                    .collect::>>()?,
-                                (true, true) => vec![],
-                            };
-                        let named_arguments = match (args.is_empty(), named_arguments.is_empty()) {
-                            (false, false) => {
-                                return Err(SparkError::invalid("conflicting named arguments"))
-                            }
-                            (false, true) => args
-                                .into_iter()
-                                .map(|(k, v)| Ok((k, spec::Expr::Literal(v.try_into()?))))
-                                .collect::>>()?,
-                            (true, false) => named_arguments
-                                .into_iter()
-                                .map(|(k, v)| Ok((k, v.try_into()?)))
-                                .collect::>>()?,
-                            (true, true) => vec![],
-                        };
                         Ok(RelationNode::Query(spec::QueryNode::WithParameters {
                             input: Box::new(input),
                             positional_arguments,
@@ -531,7 +547,7 @@ impl TryFrom for RelationNode {
                         }))
                     }
                     spec::Plan::Command(command) => {
-                        if !pos_args.is_empty() || !args.is_empty() {
+                        if !positional_arguments.is_empty() || !named_arguments.is_empty() {
                             Err(SparkError::invalid("command with parameters"))
                         } else {
                             Ok(RelationNode::Command(command.node))
@@ -672,7 +688,7 @@ impl TryFrom for RelationNode {
                 let input = input.required("with columns renamed input")?;
                 let rename_columns_map = match (rename_columns_map.is_empty(), renames.is_empty()) {
                     (false, false) => {
-                        return Err(SparkError::invalid("conflicting column renames"))
+                        return Err(SparkError::invalid("conflicting column renames"));
                     }
                     (false, true) => rename_columns_map
                         .into_iter()
@@ -893,6 +909,7 @@ impl TryFrom for RelationNode {
                     ParseFormat::Unspecified => spec::ParseFormat::Unspecified,
                     ParseFormat::Csv => spec::ParseFormat::Csv,
                     ParseFormat::Json => spec::ParseFormat::Json,
+                    ParseFormat::Xml => return Err(SparkError::unsupported("XML parse relation")),
                 };
                 let schema: Option = schema.map(|x| x.try_into()).transpose()?;
                 Ok(RelationNode::Query(spec::QueryNode::Parse(spec::Parse {
@@ -1087,10 +1104,21 @@ impl TryFrom for RelationNode {
                     .into_iter()
                     .map(|r| r.try_into())
                     .collect::>()?;
-                Ok(RelationNode::Query(spec::QueryNode::WithRelations {
-                    root: Box::new(root.try_into()?),
-                    references,
-                }))
+                let root_plan: spec::Plan = root.try_into()?;
+                match root_plan {
+                    spec::Plan::Query(query_plan) => {
+                        Ok(RelationNode::Query(spec::QueryNode::WithRelations {
+                            root: Box::new(query_plan),
+                            references,
+                        }))
+                    }
+                    spec::Plan::Command(command_plan) => {
+                        Ok(RelationNode::Command(spec::CommandNode::WithRelations {
+                            root: Box::new(command_plan),
+                            references,
+                        }))
+                    }
+                }
             }
             RelType::Transpose(_) => Err(SparkError::todo("transpose")),
             RelType::UnresolvedTableValuedFunction(_) => {
@@ -1110,7 +1138,7 @@ impl TryFrom for RelationNode {
                 let right = right.required("lateral join right")?;
                 let join_type = match JoinType::try_from(join_type)? {
                     JoinType::Unspecified => {
-                        return Err(SparkError::invalid("unspecified join type"))
+                        return Err(SparkError::invalid("unspecified join type"));
                     }
                     JoinType::Inner => spec::JoinType::Inner,
                     JoinType::FullOuter => spec::JoinType::FullOuter,
@@ -1305,6 +1333,8 @@ impl TryFrom for RelationNode {
                 }))
             }
             RelType::Catalog(catalog) => Ok(RelationNode::Command(catalog.try_into()?)),
+            RelType::RelationChanges(_) => Err(SparkError::unsupported("relation changes")),
+            RelType::NearestByJoin(_) => Err(SparkError::unsupported("nearest-by join")),
             RelType::MlRelation(_) => Err(SparkError::unsupported("ML relation")),
             RelType::Extension(_) => Err(SparkError::unsupported("extension relation")),
             RelType::Unknown(_) => Err(SparkError::unsupported("unknown relation")),
@@ -1582,6 +1612,18 @@ impl TryFrom for spec::CommandNode {
                 let sc::ListCatalogs { pattern } = x;
                 Ok(spec::CommandNode::ListCatalogs { pattern })
             }
+            CatType::DropTable(_) => Err(SparkError::unsupported("drop table")),
+            CatType::DropView(_) => Err(SparkError::unsupported("drop view")),
+            CatType::CreateDatabase(_) => Err(SparkError::unsupported("create database")),
+            CatType::DropDatabase(_) => Err(SparkError::unsupported("drop database")),
+            CatType::ListPartitions(_) => Err(SparkError::unsupported("list partitions")),
+            CatType::ListViews(_) => Err(SparkError::unsupported("list views")),
+            CatType::GetTableProperties(_) => Err(SparkError::unsupported("get table properties")),
+            CatType::GetCreateTableString(_) => {
+                Err(SparkError::unsupported("get create table string"))
+            }
+            CatType::TruncateTable(_) => Err(SparkError::unsupported("truncate table")),
+            CatType::AnalyzeTable(_) => Err(SparkError::unsupported("analyze table")),
         }
     }
 }
@@ -1625,7 +1667,11 @@ impl TryFrom for spec::Write {
             options,
             clustering_columns,
             save_type,
+            with_schema_evolution,
         } = write;
+        if with_schema_evolution {
+            return Err(SparkError::unsupported("write schema evolution"));
+        }
         let input = input.required("input")?.try_into()?;
         let mode = match SaveMode::try_from(mode).required("save mode")? {
             SaveMode::Unspecified => None,
@@ -1685,7 +1731,7 @@ impl TryFrom for spec::Write {
                 let save_method = TableSaveMethod::try_from(save_method).required("save method")?;
                 let save_method = match save_method {
                     TableSaveMethod::Unspecified => {
-                        return Err(SparkError::invalid("unspecified save method"))
+                        return Err(SparkError::invalid("unspecified save method"));
                     }
                     TableSaveMethod::SaveAsTable => spec::TableSaveMethod::SaveAsTable,
                     TableSaveMethod::InsertInto => spec::TableSaveMethod::InsertInto,
@@ -1724,7 +1770,11 @@ impl TryFrom for spec::WriteTo {
             mode,
             overwrite_condition,
             clustering_columns,
+            with_schema_evolution,
         } = write;
+        if with_schema_evolution {
+            return Err(SparkError::unsupported("write schema evolution"));
+        }
         let input = input.required("input")?.try_into()?;
         let table = from_ast_object_name(parse_object_name(table_name.as_str())?)?;
         let partitioning_columns = partitioning_columns
@@ -1882,7 +1932,7 @@ fn merge_matched_clause_from_expression(
         sc::merge_action::ActionType::Invalid
         | sc::merge_action::ActionType::Insert
         | sc::merge_action::ActionType::InsertStar => {
-            return Err(SparkError::invalid("invalid merge matched action type"))
+            return Err(SparkError::invalid("invalid merge matched action type"));
         }
     };
     Ok(spec::MergeClause::Matched(spec::MergeMatchedClause {
@@ -1911,7 +1961,7 @@ fn merge_not_matched_by_target_clause_from_expression(
         | sc::merge_action::ActionType::Delete
         | sc::merge_action::ActionType::Update
         | sc::merge_action::ActionType::UpdateStar => {
-            return Err(SparkError::invalid("invalid merge not matched action type"))
+            return Err(SparkError::invalid("invalid merge not matched action type"));
         }
     };
     Ok(spec::MergeClause::NotMatchedByTarget(
@@ -1942,7 +1992,7 @@ fn merge_not_matched_by_source_clause_from_expression(
         | sc::merge_action::ActionType::UpdateStar => {
             return Err(SparkError::invalid(
                 "invalid merge not matched by source action type",
-            ))
+            ));
         }
     };
     Ok(spec::MergeClause::NotMatchedBySource(
@@ -2138,7 +2188,7 @@ mod tests {
     #[test]
     fn test_merge_into_table_command_to_plan() -> SparkResult<()> {
         use sc::expression::{ExprType, ExpressionString, UnresolvedAttribute};
-        use sc::{merge_action, relation, MergeAction};
+        use sc::{MergeAction, merge_action, relation};
 
         let source_table_plan = sc::Relation {
             common: None,
diff --git a/crates/sail-spark-connect/src/schema.rs b/crates/sail-spark-connect/src/schema.rs
index dc18eff30..55dfa6fac 100644
--- a/crates/sail-spark-connect/src/schema.rs
+++ b/crates/sail-spark-connect/src/schema.rs
@@ -2,7 +2,7 @@ use std::fmt;
 use std::fmt::Display;
 
 use datafusion::arrow::datatypes::{DataType, SchemaRef};
-use sail_common::string::escape_meta_characters;
+use sail_common::utils::string::escape_meta_characters;
 
 use crate::error::{SparkError, SparkResult};
 use crate::spark::connect as sc;
diff --git a/crates/sail-spark-connect/src/server.rs b/crates/sail-spark-connect/src/server.rs
index 9c47c96bf..3032f8d1b 100644
--- a/crates/sail-spark-connect/src/server.rs
+++ b/crates/sail-spark-connect/src/server.rs
@@ -15,12 +15,12 @@ use crate::spark::connect::interrupt_request::{Interrupt, InterruptType};
 use crate::spark::connect::release_execute_request::{Release, ReleaseAll, ReleaseUntil};
 use crate::spark::connect::spark_connect_service_server::SparkConnectService;
 use crate::spark::connect::{
-    config_request, plan, AddArtifactsRequest, AddArtifactsResponse, AnalyzePlanRequest,
-    AnalyzePlanResponse, ArtifactStatusesRequest, ArtifactStatusesResponse, CloneSessionRequest,
-    CloneSessionResponse, Command, ConfigRequest, ConfigResponse, ExecutePlanRequest,
-    FetchErrorDetailsRequest, FetchErrorDetailsResponse, InterruptRequest, InterruptResponse, Plan,
-    ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse, ReleaseSessionRequest,
-    ReleaseSessionResponse,
+    AddArtifactsRequest, AddArtifactsResponse, AnalyzePlanRequest, AnalyzePlanResponse,
+    ArtifactStatusesRequest, ArtifactStatusesResponse, CloneSessionRequest, CloneSessionResponse,
+    Command, ConfigRequest, ConfigResponse, ExecutePlanRequest, FetchErrorDetailsRequest,
+    FetchErrorDetailsResponse, GetStatusRequest, GetStatusResponse, InterruptRequest,
+    InterruptResponse, Plan, ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse,
+    ReleaseSessionRequest, ReleaseSessionResponse, config_request, plan,
 };
 
 #[derive(Debug)]
@@ -100,8 +100,9 @@ async fn handle_command(
         CommandType::CheckpointCommand(checkpoint) => {
             service::handle_execute_checkpoint_command(ctx, checkpoint, metadata).await
         }
-        CommandType::RemoveCachedRemoteRelationCommand(_) => {
-            Err(SparkError::todo("remove cached remote relation command"))
+        CommandType::RemoveCachedRemoteRelationCommand(command) => {
+            service::handle_execute_remove_cached_remote_relation_command(ctx, command, metadata)
+                .await
         }
         CommandType::MergeIntoTableCommand(command) => {
             service::handle_execute_merge_into_table_command(ctx, command, metadata).await
@@ -484,4 +485,12 @@ impl SparkConnectService for SparkConnectServer {
         debug!("{request:?}");
         Err(Status::unimplemented("clone session"))
     }
+
+    async fn get_status(
+        &self,
+        request: Request,
+    ) -> Result, Status> {
+        debug!("{:?}", request.into_inner());
+        Err(Status::unimplemented("get status"))
+    }
 }
diff --git a/crates/sail-spark-connect/src/service/artifact_manager.rs b/crates/sail-spark-connect/src/service/artifact_manager.rs
index 04b586810..2d2c7d7a4 100644
--- a/crates/sail-spark-connect/src/service/artifact_manager.rs
+++ b/crates/sail-spark-connect/src/service/artifact_manager.rs
@@ -1,8 +1,8 @@
 use std::collections::HashMap;
 
 use datafusion::prelude::SessionContext;
-use tonic::codegen::tokio_stream::Stream;
 use tonic::Status;
+use tonic::codegen::tokio_stream::Stream;
 
 use crate::error::{SparkError, SparkResult};
 use crate::spark::connect::add_artifacts_request::Payload;
diff --git a/crates/sail-spark-connect/src/service/config_manager.rs b/crates/sail-spark-connect/src/service/config_manager.rs
index de13021c8..c0cbc94aa 100644
--- a/crates/sail-spark-connect/src/service/config_manager.rs
+++ b/crates/sail-spark-connect/src/service/config_manager.rs
@@ -1,7 +1,7 @@
 use datafusion::prelude::SessionContext;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 
-use crate::config::{ConfigKeyValue, SparkRuntimeConfig};
+use crate::config::ConfigKeyValue;
 use crate::error::SparkResult;
 use crate::session::SparkSession;
 use crate::spark::connect::{ConfigResponse, KeyValue};
@@ -11,7 +11,7 @@ pub(crate) fn handle_config_get(
     keys: Vec,
 ) -> SparkResult {
     let spark = ctx.extension::()?;
-    let warnings = SparkRuntimeConfig::get_warnings_by_keys(&keys);
+    let warnings = spark.get_config_warnings_by_keys(&keys)?;
     let pairs = spark.get_config(keys)?;
     let pairs = pairs.into_iter().map(Into::into).collect();
     Ok(ConfigResponse {
@@ -28,7 +28,7 @@ pub(crate) fn handle_config_set(
 ) -> SparkResult {
     let spark = ctx.extension::()?;
     let kv: Vec = kv.into_iter().map(Into::into).collect();
-    let warnings = SparkRuntimeConfig::get_warnings(&kv);
+    let warnings = spark.get_config_warnings(&kv)?;
     spark.set_config(kv)?;
     Ok(ConfigResponse {
         session_id: spark.session_id().to_string(),
@@ -44,7 +44,7 @@ pub(crate) fn handle_config_get_with_default(
 ) -> SparkResult {
     let spark = ctx.extension::()?;
     let kv: Vec = kv.into_iter().map(Into::into).collect();
-    let warnings = SparkRuntimeConfig::get_warnings(&kv);
+    let warnings = spark.get_config_warnings(&kv)?;
     let pairs = spark.get_config_with_default(kv)?;
     let pairs = pairs.into_iter().map(Into::into).collect();
     Ok(ConfigResponse {
@@ -60,7 +60,7 @@ pub(crate) fn handle_config_get_option(
     keys: Vec,
 ) -> SparkResult {
     let spark = ctx.extension::()?;
-    let warnings = SparkRuntimeConfig::get_warnings_by_keys(&keys);
+    let warnings = spark.get_config_warnings_by_keys(&keys)?;
     let pairs = spark.get_config_option(keys)?;
     let pairs = pairs.into_iter().map(Into::into).collect();
     Ok(ConfigResponse {
@@ -77,7 +77,7 @@ pub(crate) fn handle_config_get_all(
 ) -> SparkResult {
     let spark = ctx.extension::()?;
     let kv = spark.get_all_config(prefix.as_deref())?;
-    let warnings = SparkRuntimeConfig::get_warnings(&kv);
+    let warnings = spark.get_config_warnings(&kv)?;
     let pairs = kv.into_iter().map(Into::into).collect();
     Ok(ConfigResponse {
         session_id: spark.session_id().to_string(),
@@ -92,7 +92,7 @@ pub(crate) fn handle_config_unset(
     keys: Vec,
 ) -> SparkResult {
     let spark = ctx.extension::()?;
-    let warnings = SparkRuntimeConfig::get_warnings_by_keys(&keys);
+    let warnings = spark.get_config_warnings_by_keys(&keys)?;
     spark.unset_config(keys)?;
     Ok(ConfigResponse {
         session_id: spark.session_id().to_string(),
@@ -107,18 +107,18 @@ pub(crate) fn handle_config_is_modifiable(
     keys: Vec,
 ) -> SparkResult {
     let spark = ctx.extension::()?;
-    let warnings = SparkRuntimeConfig::get_warnings_by_keys(&keys);
+    let warnings = spark.get_config_warnings_by_keys(&keys)?;
     let pairs = keys
         .into_iter()
-        .map(|key| {
-            let modifiable = SparkRuntimeConfig::is_modifiable(key.as_str());
+        .map(|key| -> SparkResult<_> {
+            let modifiable = spark.is_config_modifiable(key.as_str())?;
             let value = if modifiable { "true" } else { "false" };
-            KeyValue {
+            Ok(KeyValue {
                 key: key.clone(),
                 value: Some(value.to_string()),
-            }
+            })
         })
-        .collect();
+        .collect::>>()?;
     Ok(ConfigResponse {
         session_id: spark.session_id().to_string(),
         server_side_session_id: spark.session_id().to_string(),
diff --git a/crates/sail-spark-connect/src/service/plan_analyzer.rs b/crates/sail-spark-connect/src/service/plan_analyzer.rs
index cde3b100f..a15ac9c01 100644
--- a/crates/sail-spark-connect/src/service/plan_analyzer.rs
+++ b/crates/sail-spark-connect/src/service/plan_analyzer.rs
@@ -3,9 +3,9 @@ use log::warn;
 use sail_common::spec;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::rename::schema::rename_schema;
-use sail_plan::explain::{explain_string, ExplainOptions};
-use sail_plan::resolver::plan::NamedPlan;
+use sail_plan::explain::{ExplainOptions, explain_string};
 use sail_plan::resolver::PlanResolver;
+use sail_plan::resolver::plan::NamedPlan;
 
 use crate::config::get_pyspark_version;
 use crate::error::{ProtoFieldExt, SparkError, SparkResult};
@@ -31,7 +31,7 @@ use crate::spark::connect::analyze_plan_response::{
     SemanticHash as SemanticHashResponse, SparkVersion as SparkVersionResponse,
     TreeString as TreeStringResponse, Unpersist as UnpersistResponse,
 };
-use crate::spark::connect::{plan, StorageLevel};
+use crate::spark::connect::{StorageLevel, plan};
 
 async fn analyze_schema(ctx: &SessionContext, plan: sc::Plan) -> SparkResult {
     let spark = ctx.extension::()?;
@@ -114,10 +114,18 @@ pub(crate) async fn handle_analyze_is_streaming(
 }
 
 pub(crate) async fn handle_analyze_input_files(
-    _ctx: &SessionContext,
-    _request: InputFilesRequest,
+    ctx: &SessionContext,
+    request: InputFilesRequest,
 ) -> SparkResult {
-    Err(SparkError::todo("handle analyze input files"))
+    let InputFilesRequest { plan } = request;
+    let plan = plan.required("plan")?;
+    let spark = ctx.extension::()?;
+    let resolver = PlanResolver::new(ctx, spark.plan_config()?);
+    let NamedPlan { plan, .. } = resolver
+        .resolve_named_plan(spec::Plan::Query(plan.try_into()?))
+        .await?;
+    let files = sail_data_source::listing::input_files::input_files(ctx, plan).await?;
+    Ok(InputFilesResponse { files })
 }
 
 pub(crate) async fn handle_analyze_spark_version(
diff --git a/crates/sail-spark-connect/src/service/plan_executor.rs b/crates/sail-spark-connect/src/service/plan_executor.rs
index bf2a30e61..b86a50ae6 100644
--- a/crates/sail-spark-connect/src/service/plan_executor.rs
+++ b/crates/sail-spark-connect/src/service/plan_executor.rs
@@ -3,36 +3,36 @@ use std::task::{Context, Poll};
 
 use datafusion::arrow::compute::concat_batches;
 use datafusion::prelude::SessionContext;
+use fastrace::Span;
 use fastrace::collector::SpanContext;
 use fastrace::future::FutureExt;
-use fastrace::Span;
 use futures::stream;
 use log::{debug, warn};
 use sail_common::spec;
 use sail_common_datafusion::extension::SessionExtensionAccessor;
 use sail_common_datafusion::session::job::JobService;
 use sail_plan::resolve_and_execute_plan;
-use tonic::codegen::tokio_stream::wrappers::ReceiverStream;
-use tonic::codegen::tokio_stream::Stream;
 use tonic::Status;
+use tonic::codegen::tokio_stream::Stream;
 
 use crate::error::{ProtoFieldExt, SparkError, SparkResult};
 use crate::executor::{
-    read_stream, to_arrow_batch, Executor, ExecutorBatch, ExecutorMetadata, ExecutorOutput,
-    ExecutorOutputStream,
+    Executor, ExecutorBatch, ExecutorMetadata, ExecutorMode, ExecutorOutput, ExecutorOutputStream,
+    to_arrow_batch,
 };
 use crate::session::SparkSession;
 use crate::spark::connect::execute_plan_response::{
     ResponseType, ResultComplete, SqlCommandResult,
 };
 use crate::spark::connect::{
-    relation, CheckpointCommand, CheckpointCommandResult, CommonInlineUserDefinedDataSource,
-    CommonInlineUserDefinedFunction, CommonInlineUserDefinedTableFunction,
-    CreateDataFrameViewCommand, ExecutePlanResponse, GetResourcesCommand, LocalRelation,
-    MergeIntoTableCommand, Relation, SqlCommand, StreamingQueryCommand,
+    CachedRemoteRelation, CheckpointCommand, CheckpointCommandResult,
+    CommonInlineUserDefinedDataSource, CommonInlineUserDefinedFunction,
+    CommonInlineUserDefinedTableFunction, CreateDataFrameViewCommand, ExecutePlanResponse,
+    GetResourcesCommand, LocalRelation, MergeIntoTableCommand, Relation,
+    RemoveCachedRemoteRelationCommand, SqlCommand, StreamingQueryCommand,
     StreamingQueryCommandResult, StreamingQueryListenerBusCommand, StreamingQueryManagerCommand,
     StreamingQueryManagerCommandResult, WriteOperation, WriteOperationV2,
-    WriteStreamOperationStart, WriteStreamOperationStartResult,
+    WriteStreamOperationStart, WriteStreamOperationStartResult, relation,
 };
 use crate::streaming::timeout_millis;
 
@@ -61,12 +61,14 @@ impl Stream for ExecutePlanResponseStream {
     ) -> Poll>> {
         self.inner.as_mut().poll_next(cx).map(|poll| {
             poll.map(|item| {
+                let item = item.map_err(Status::from)?;
                 let mut response = ExecutePlanResponse::default();
                 response.session_id.clone_from(&self.session_id);
                 response.server_side_session_id.clone_from(&self.session_id);
                 response.operation_id.clone_from(&self.operation_id.clone());
                 response.response_id = item.id;
                 match item.batch {
+                    ExecutorBatch::Heartbeat => {}
                     ExecutorBatch::ArrowBatch(batch) => {
                         response.response_type = Some(ResponseType::ArrowBatch(batch));
                     }
@@ -108,19 +110,11 @@ impl Stream for ExecutePlanResponseStream {
     }
 }
 
-enum ExecutePlanMode {
-    /// Execute the plan lazily as the client reads the response stream.
-    Lazy,
-    /// Execute the plan eagerly and return an empty response stream.
-    /// This is useful for executing command plans.
-    EagerSilent,
-}
-
 async fn handle_execute_plan(
     ctx: &SessionContext,
     plan: spec::Plan,
     metadata: ExecutorMetadata,
-    mode: ExecutePlanMode,
+    mode: ExecutorMode,
 ) -> SparkResult {
     let span = Span::root("handle_execute_plan", SpanContext::random());
     let spark = ctx.extension::()?;
@@ -131,31 +125,19 @@ async fn handle_execute_plan(
         let span = Span::enter_with_parent("JobRunner::execute", &span);
         service.runner().execute(ctx, plan).in_span(span).await?
     };
-    let rx = match mode {
-        ExecutePlanMode::Lazy => {
-            let _guard = span.set_local_parent();
-            let executor = Executor::new(
-                metadata,
-                stream,
-                spark.options().execution_heartbeat_interval,
-            );
-            let rx = executor.start()?;
-            spark.add_executor(executor)?;
-            rx
-        }
-        ExecutePlanMode::EagerSilent => {
-            let _ = read_stream(stream).in_span(span).await?;
-            let (tx, rx) = tokio::sync::mpsc::channel(1);
-            if metadata.reattachable {
-                tx.send(ExecutorOutput::complete()).await?;
-            }
-            ReceiverStream::new(rx)
-        }
-    };
+    let _guard = span.set_local_parent();
+    let executor = Executor::new(
+        metadata,
+        stream,
+        spark.options().execution_heartbeat_interval,
+        mode,
+    );
+    let rx = executor.start()?;
+    spark.add_executor(executor)?;
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         operation_id,
-        Box::pin(rx),
+        rx,
     ))
 }
 
@@ -165,7 +147,7 @@ pub(crate) async fn handle_execute_relation(
     metadata: ExecutorMetadata,
 ) -> SparkResult {
     let plan = relation.try_into()?;
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::Lazy).await
+    handle_execute_plan(ctx, plan, metadata, ExecutorMode::Query).await
 }
 
 pub(crate) async fn handle_execute_register_function(
@@ -176,7 +158,8 @@ pub(crate) async fn handle_execute_register_function(
     let plan = spec::Plan::Command(spec::CommandPlan::new(spec::CommandNode::RegisterFunction(
         udf.try_into()?,
     )));
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await
+    let mode = ExecutorMode::command();
+    handle_execute_plan(ctx, plan, metadata, mode).await
 }
 
 pub(crate) async fn handle_execute_write_operation(
@@ -187,7 +170,8 @@ pub(crate) async fn handle_execute_write_operation(
     let plan = spec::Plan::Command(spec::CommandPlan::new(spec::CommandNode::Write(
         write.try_into()?,
     )));
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await
+    let mode = ExecutorMode::command();
+    handle_execute_plan(ctx, plan, metadata, mode).await
 }
 
 pub(crate) async fn handle_execute_create_dataframe_view(
@@ -196,7 +180,8 @@ pub(crate) async fn handle_execute_create_dataframe_view(
     metadata: ExecutorMetadata,
 ) -> SparkResult {
     let plan = spec::Plan::Command(spec::CommandPlan::new(view.try_into()?));
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await
+    let mode = ExecutorMode::command();
+    handle_execute_plan(ctx, plan, metadata, mode).await
 }
 
 pub(crate) async fn handle_execute_write_operation_v2(
@@ -207,7 +192,8 @@ pub(crate) async fn handle_execute_write_operation_v2(
     let plan = spec::Plan::Command(spec::CommandPlan::new(spec::CommandNode::WriteTo(
         write.try_into()?,
     )));
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await
+    let mode = ExecutorMode::command();
+    handle_execute_plan(ctx, plan, metadata, mode).await
 }
 
 pub(crate) async fn handle_execute_merge_into_table_command(
@@ -216,7 +202,8 @@ pub(crate) async fn handle_execute_merge_into_table_command(
     metadata: ExecutorMetadata,
 ) -> SparkResult {
     let plan = spec::Plan::Command(spec::CommandPlan::new(command.try_into()?));
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await
+    let mode = ExecutorMode::command();
+    handle_execute_plan(ctx, plan, metadata, mode).await
 }
 
 /// Handles execution of a SQL command.
@@ -227,7 +214,6 @@ pub(crate) async fn handle_execute_sql_command(
     metadata: ExecutorMetadata,
 ) -> SparkResult {
     let spark = ctx.extension::()?;
-    let service = ctx.extension::()?;
     let relation = if let Some(input) = sql.input {
         input
     } else {
@@ -244,35 +230,40 @@ pub(crate) async fn handle_execute_sql_command(
         }
     };
     let plan: spec::Plan = relation.clone().try_into()?;
-    let relation = match plan {
-        spec::Plan::Query(_) => relation,
-        command @ spec::Plan::Command(_) => {
-            let (plan, _) = resolve_and_execute_plan(ctx, spark.plan_config()?, command).await?;
-            let stream = service.runner().execute(ctx, plan).await?;
-            let schema = stream.schema();
-            let data = read_stream(stream).await?;
-            let data = concat_batches(&schema, data.iter())?;
-            Relation {
-                common: None,
-                rel_type: Some(relation::RelType::LocalRelation(LocalRelation {
-                    data: Some(to_arrow_batch(&data)?.data),
-                    schema: None,
-                })),
+    match plan {
+        spec::Plan::Command(command) => {
+            let mode = ExecutorMode::command_with_completion(move |schema, data| {
+                let data = concat_batches(&schema, data.iter())?;
+                let relation = Relation {
+                    common: None,
+                    rel_type: Some(relation::RelType::LocalRelation(LocalRelation {
+                        data: Some(to_arrow_batch(&data)?.data),
+                        schema: None,
+                    })),
+                };
+                Ok(Some(ExecutorOutput::new(ExecutorBatch::SqlCommandResult(
+                    Box::new(SqlCommandResult {
+                        relation: Some(relation),
+                    }),
+                ))))
+            });
+            handle_execute_plan(ctx, spec::Plan::Command(command), metadata, mode).await
+        }
+        spec::Plan::Query(_) => {
+            let result = ExecutorBatch::SqlCommandResult(Box::new(SqlCommandResult {
+                relation: Some(relation),
+            }));
+            let mut output = vec![ExecutorOutput::new(result)];
+            if metadata.reattachable {
+                output.push(ExecutorOutput::complete());
             }
+            Ok(ExecutePlanResponseStream::new(
+                spark.session_id().to_string(),
+                metadata.operation_id,
+                Box::pin(stream::iter(output.into_iter().map(Ok))),
+            ))
         }
-    };
-    let result = ExecutorBatch::SqlCommandResult(Box::new(SqlCommandResult {
-        relation: Some(relation),
-    }));
-    let mut output = vec![ExecutorOutput::new(result)];
-    if metadata.reattachable {
-        output.push(ExecutorOutput::complete());
     }
-    Ok(ExecutePlanResponseStream::new(
-        spark.session_id().to_string(),
-        metadata.operation_id,
-        Box::pin(stream::iter(output)),
-    ))
 }
 
 pub(crate) async fn handle_execute_write_stream_operation_start(
@@ -304,7 +295,7 @@ pub(crate) async fn handle_execute_write_stream_operation_start(
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         operation_id,
-        Box::pin(stream::iter(output)),
+        Box::pin(stream::iter(output.into_iter().map(Ok))),
     ))
 }
 
@@ -389,7 +380,7 @@ pub(crate) async fn handle_execute_streaming_query_command(
         | Command::Exception(false) => {
             return Err(SparkError::invalid(format!(
                 "invalid streaming query command: {command:?}"
-            )))
+            )));
         }
     };
     let result = StreamingQueryCommandResult {
@@ -405,7 +396,7 @@ pub(crate) async fn handle_execute_streaming_query_command(
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         metadata.operation_id,
-        Box::pin(stream::iter(output)),
+        Box::pin(stream::iter(output.into_iter().map(Ok))),
     ))
 }
 
@@ -464,18 +455,18 @@ pub(crate) async fn handle_execute_streaming_query_manager_command(
             Some(ResultType::ResetTerminated(true))
         }
         Command::AddListener(_) => {
-            return Err(SparkError::NotImplemented("add listener".to_string()))
+            return Err(SparkError::NotImplemented("add listener".to_string()));
         }
         Command::RemoveListener(_) => {
-            return Err(SparkError::NotImplemented("remove listener".to_string()))
+            return Err(SparkError::NotImplemented("remove listener".to_string()));
         }
         Command::ListListeners(_) => {
-            return Err(SparkError::NotImplemented("list listeners".to_string()))
+            return Err(SparkError::NotImplemented("list listeners".to_string()));
         }
         Command::Active(false) | Command::ResetTerminated(false) => {
             return Err(SparkError::invalid(format!(
                 "invalid streaming query manager command: {command:?}"
-            )))
+            )));
         }
     };
     let result = StreamingQueryManagerCommandResult { result_type };
@@ -488,7 +479,7 @@ pub(crate) async fn handle_execute_streaming_query_manager_command(
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         metadata.operation_id,
-        Box::pin(stream::iter(output)),
+        Box::pin(stream::iter(output.into_iter().map(Ok))),
     ))
 }
 
@@ -500,7 +491,8 @@ pub(crate) async fn handle_execute_register_table_function(
     let plan = spec::Plan::Command(spec::CommandPlan::new(
         spec::CommandNode::RegisterTableFunction(udtf.try_into()?),
     ));
-    handle_execute_plan(ctx, plan, metadata, ExecutePlanMode::EagerSilent).await
+    let mode = ExecutorMode::command();
+    handle_execute_plan(ctx, plan, metadata, mode).await
 }
 
 pub(crate) async fn handle_execute_streaming_query_listener_bus_command(
@@ -515,23 +507,59 @@ pub(crate) async fn handle_execute_streaming_query_listener_bus_command(
 
 pub(crate) async fn handle_execute_checkpoint_command(
     ctx: &SessionContext,
-    _checkpoint: CheckpointCommand,
+    checkpoint: CheckpointCommand,
     metadata: ExecutorMetadata,
 ) -> SparkResult {
-    // TODO: Implement
-    warn!("Checkpoint operation is not yet supported and is a no-op");
-    let spark = ctx.extension::()?;
-    let result = CheckpointCommandResult { relation: None };
-    let mut output = vec![ExecutorOutput::new(ExecutorBatch::CheckpointCommandResult(
-        Box::new(result),
-    ))];
-    if metadata.reattachable {
-        output.push(ExecutorOutput::complete());
+    let CheckpointCommand {
+        relation,
+        local: _,
+        eager,
+        storage_level,
+    } = checkpoint;
+    if !eager {
+        return Err(SparkError::unsupported("lazy DataFrame checkpoint"));
+    }
+    if storage_level.is_some() {
+        return Err(SparkError::unsupported(
+            "checkpoint StorageLevel; Sail checkpoints are object-store backed",
+        ));
     }
+    let relation = relation.required("checkpoint relation")?;
+    let query: spec::QueryPlan = relation.try_into()?;
+    let relation_id = uuid::Uuid::new_v4().to_string();
+    let plan = spec::Plan::Command(spec::CommandPlan::new(
+        spec::CommandNode::RemoteCheckpoint {
+            relation_id: relation_id.clone(),
+            input: Box::new(query),
+        },
+    ));
+    let mode = ExecutorMode::command_with_completion(move |_, _| {
+        Ok(Some(ExecutorOutput::new(
+            ExecutorBatch::CheckpointCommandResult(Box::new(CheckpointCommandResult {
+                relation: Some(CachedRemoteRelation { relation_id }),
+            })),
+        )))
+    });
+    handle_execute_plan(ctx, plan, metadata, mode).await
+}
+
+pub(crate) async fn handle_execute_remove_cached_remote_relation_command(
+    ctx: &SessionContext,
+    _command: RemoveCachedRemoteRelationCommand,
+    metadata: ExecutorMetadata,
+) -> SparkResult {
+    let spark = ctx.extension::()?;
+    // TODO: Remove checkpoint data on a best-effort basis when the client releases the relation.
+    // Checkpoints have session lifetime so plans can safely share their immutable relation ID.
+    // The registry and object-store namespace are cleared together after the job runner stops.
+    let output = metadata
+        .reattachable
+        .then(ExecutorOutput::complete)
+        .into_iter();
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         metadata.operation_id,
-        Box::pin(stream::iter(output)),
+        Box::pin(stream::iter(output.into_iter().map(Ok))),
     ))
 }
 
@@ -587,12 +615,14 @@ pub(crate) async fn handle_reattach_execute(
         )));
     }
     executor.pause_if_running().await?;
-    executor.release(response_id)?;
+    if let Some(response_id) = response_id {
+        executor.release(response_id)?;
+    }
     let rx = executor.start()?;
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         operation_id,
-        Box::pin(rx),
+        rx,
     ))
 }
 
@@ -602,10 +632,15 @@ pub(crate) async fn handle_release_execute(
     response_id: Option,
 ) -> SparkResult<()> {
     let spark = ctx.extension::()?;
-    // Some operations may not have an executor (e.g. DDL statements),
-    // so it is a no-op if the executor is not found.
-    if let Some(executor) = spark.get_executor(operation_id.as_str())? {
+    let executor = spark.get_executor(operation_id.as_str())?;
+    // TODO: clean up non-reattachable executors which the client does not release explicitly
+    let Some(executor) = executor.filter(|executor| executor.metadata.reattachable) else {
+        return Ok(());
+    };
+    if let Some(response_id) = response_id {
         executor.release(response_id)?;
+    } else if let Some(executor) = spark.remove_executor(operation_id.as_str())? {
+        executor.pause_if_running().await?;
     }
     Ok(())
 }
@@ -631,7 +666,7 @@ pub(crate) async fn handle_execute_register_datasource(
         None => {
             return Err(SparkError::invalid(
                 "RegisterDataSource requires a python_data_source",
-            ))
+            ));
         }
     };
 
@@ -644,17 +679,20 @@ pub(crate) async fn handle_execute_register_datasource(
 
         // Register format in session's TableFormatRegistry with embedded pickled class
         // This provides session isolation - the format is only visible to this session
-        if let Ok(registry) = ctx.extension::() {
-            let format = Arc::new(PythonTableFormat::with_pickled_class(name.clone(), command));
-            // Ignore error if already registered (allows re-registration to update)
-            if let Err(e) = registry.register(format) {
-                warn!("Failed to register python datasource {}: {}", name, e);
+        match ctx.extension::() {
+            Ok(registry) => {
+                let format = Arc::new(PythonTableFormat::with_pickled_class(name.clone(), command));
+                // Ignore error if already registered (allows re-registration to update)
+                if let Err(e) = registry.register(format) {
+                    warn!("Failed to register python datasource {}: {}", name, e);
+                }
+                log::info!("Registered session-scoped datasource: {}", name);
+            }
+            _ => {
+                return Err(SparkError::internal(
+                    "TableFormatRegistry not found in session context",
+                ));
             }
-            log::info!("Registered session-scoped datasource: {}", name);
-        } else {
-            return Err(SparkError::internal(
-                "TableFormatRegistry not found in session context",
-            ));
         }
     }
 
@@ -666,6 +704,6 @@ pub(crate) async fn handle_execute_register_datasource(
     Ok(ExecutePlanResponseStream::new(
         spark.session_id().to_string(),
         metadata.operation_id,
-        Box::pin(stream::iter(output)),
+        Box::pin(stream::iter(output.into_iter().map(Ok))),
     ))
 }
diff --git a/crates/sail-spark-connect/src/session.rs b/crates/sail-spark-connect/src/session.rs
index ef70593f6..277060435 100644
--- a/crates/sail-spark-connect/src/session.rs
+++ b/crates/sail-spark-connect/src/session.rs
@@ -5,14 +5,14 @@ use std::time::Duration;
 
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::logical_expr::StringifiedPlan;
-use sail_common::datetime::get_system_timezone;
+use sail_common::utils::datetime::get_system_timezone;
 use sail_common_datafusion::extension::SessionExtension;
 use sail_plan::config::PlanConfig;
 
 use crate::config::{ConfigKeyValue, SparkRuntimeConfig};
 use crate::error::{SparkError, SparkResult, SparkThrowable};
 use crate::executor::Executor;
-use crate::spark::config::SPARK_SQL_SESSION_TIME_ZONE;
+use crate::spark::config::SparkConfigKey;
 use crate::streaming::{
     StreamingQuery, StreamingQueryAwaitHandle, StreamingQueryAwaitHandleSet, StreamingQueryId,
     StreamingQueryManager, StreamingQueryStatus,
@@ -59,10 +59,10 @@ impl SparkSession {
             session_id,
             user_id,
             options,
-            state: Mutex::new(SparkSessionState::new()),
+            state: Mutex::new(SparkSessionState::try_new()?),
         };
         extension.set_config(vec![ConfigKeyValue {
-            key: SPARK_SQL_SESSION_TIME_ZONE.to_string(),
+            key: SparkConfigKey::SPARK_SQL_SESSION_TIME_ZONE.to_string(),
             value: Some(get_system_timezone()?),
         }])?;
         Ok(extension)
@@ -154,6 +154,21 @@ impl SparkSession {
         state.config.get_all(prefix)
     }
 
+    pub(crate) fn get_config_warnings(&self, kv: &[ConfigKeyValue]) -> SparkResult> {
+        let state = self.state.lock()?;
+        Ok(state.config.get_warnings(kv))
+    }
+
+    pub(crate) fn get_config_warnings_by_keys(&self, keys: &[String]) -> SparkResult> {
+        let state = self.state.lock()?;
+        Ok(state.config.get_warnings_by_keys(keys))
+    }
+
+    pub(crate) fn is_config_modifiable(&self, key: &str) -> SparkResult {
+        let state = self.state.lock()?;
+        Ok(state.config.is_modifiable(key))
+    }
+
     pub(crate) fn add_executor(&self, executor: Executor) -> SparkResult<()> {
         let mut state = self.state.lock()?;
         let id = executor.metadata.operation_id.clone();
@@ -297,11 +312,11 @@ struct SparkSessionState {
 }
 
 impl SparkSessionState {
-    fn new() -> Self {
-        Self {
-            config: SparkRuntimeConfig::new(),
+    fn try_new() -> SparkResult {
+        Ok(Self {
+            config: SparkRuntimeConfig::try_new()?,
             executors: HashMap::new(),
             streaming_queries: StreamingQueryManager::new(),
-        }
+        })
     }
 }
diff --git a/crates/sail-spark-connect/src/session_manager.rs b/crates/sail-spark-connect/src/session_manager.rs
index b94582dbb..60aac4238 100644
--- a/crates/sail-spark-connect/src/session_manager.rs
+++ b/crates/sail-spark-connect/src/session_manager.rs
@@ -1,23 +1,23 @@
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use std::time::Duration;
 
-use datafusion::common::{internal_datafusion_err, Result};
-use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::common::{Result, internal_datafusion_err};
 use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
 use datafusion::prelude::SessionConfig;
+use sail_common::actor::ActorSystem;
 use sail_common::config::AppConfig;
 use sail_common::runtime::RuntimeHandle;
 use sail_common_datafusion::catalog::display::DefaultCatalogDisplay;
 use sail_common_datafusion::session::plan::PlanService;
 use sail_plan::catalog::SparkCatalogObjectDisplay;
 use sail_plan::formatter::SparkPlanFormatter;
-use sail_server::actor::ActorSystem;
 use sail_session::session_factory::{
     ServerSessionFactory, ServerSessionInfo, ServerSessionMutator, SessionFactory,
 };
-use sail_session::session_manager::{SessionManager, SessionManagerOptions};
+use sail_session::session_manager::{SessionManager, create_session_manager};
 
-use crate::error::{SparkError, SparkResult};
+use crate::error::SparkResult;
 use crate::session::{SparkSession, SparkSessionOptions};
 
 pub struct SparkSessionMutator {
@@ -69,29 +69,24 @@ impl ServerSessionMutator for SparkSessionMutator {
 fn create_spark_session_factory(
     config: Arc,
     runtime: RuntimeHandle,
-    system: Arc>,
 ) -> Box> {
     let mutator = Box::new(SparkSessionMutator {
         config: config.clone(),
     });
-    Box::new(ServerSessionFactory::new(config, runtime, system, mutator))
+    Box::new(ServerSessionFactory::new(config, runtime, mutator))
 }
 
-pub fn create_spark_session_manager(
+pub async fn create_spark_session_manager(
     config: Arc,
     runtime: RuntimeHandle,
+    system: &mut ActorSystem,
 ) -> SparkResult {
-    let system = Arc::new(Mutex::new(ActorSystem::new()));
-    let factory = {
-        let config = config.clone();
-        let runtime = runtime.clone();
-        let system = system.clone();
-        Box::new(move || {
-            create_spark_session_factory(config.clone(), runtime.clone(), system.clone())
-        })
-    };
-    let options = SessionManagerOptions::new(runtime.clone(), system, factory)
-        .with_session_timeout(Duration::from_secs(config.spark.session_timeout_secs))
-        .with_options(config.raw().map_err(SparkError::from)?);
-    Ok(SessionManager::try_new(options)?)
+    Ok(create_session_manager(
+        config.clone(),
+        runtime,
+        create_spark_session_factory,
+        Duration::from_secs(config.spark.session_timeout_secs),
+        system,
+    )
+    .await?)
 }
diff --git a/crates/sail-spark-connect/tests/gold_data/README.md b/crates/sail-spark-connect/tests/gold_data/README.md
deleted file mode 100644
index 1d2851500..000000000
--- a/crates/sail-spark-connect/tests/gold_data/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Gold Data for Spark Tests
-
-This directory contains the gold data for the Spark tests.
-All the files in this directory, including this README file, are auto-generated.
-Please do not modify them manually.
diff --git a/crates/sail-spark-connect/tests/gold_data/data_type.json b/crates/sail-spark-connect/tests/gold_data/data_type.json
index 0f154a877..ea7267a7d 100644
--- a/crates/sail-spark-connect/tests/gold_data/data_type.json
+++ b/crates/sail-spark-connect/tests/gold_data/data_type.json
@@ -362,7 +362,18 @@
     {
       "input": "TIME(6) WITHOUT TIME ZONE",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found ( at 11:12 expected ':', or data type"
+      }
+    },
+    {
+      "input": "TIMESTAMP WITH LOCAL TIME ZONE",
+      "output": {
+        "success": {
+          "timestamp": {
+            "timeUnit": "microsecond",
+            "timestampType": "withLocalTimeZone"
+          }
+        }
       }
     },
     {
@@ -389,7 +400,7 @@
     {
       "input": "TimE WiTHOUT TiME ZoNE",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found WiTHOUT at 12:19 expected ':', or data type"
       }
     },
     {
@@ -530,7 +541,7 @@
       "input": "it is not a data type",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'is'. SQLSTATE: 42601 (line 1, pos 3)\n\n== SQL ==\nit is not a data type\n---^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found is at 10:12 expected ':', or data type"
       }
     },
     {
@@ -692,28 +703,28 @@
       "input": "struct",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near '<'. SQLSTATE: 42601 (line 1, pos 6)\n\n== SQL ==\nstruct\n------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found < at 13:14 expected ':', or data type"
       }
     },
     {
       "input": "struct at 14:15 expected ':', or data type"
       }
     },
     {
       "input": "unknown(1,2,3)",
       "exception": "\n[UNSUPPORTED_DATATYPE] Unsupported data type \"UNKNOWN(1,2,3)\". SQLSTATE: 0A000 (line 1, pos 0)\n\n== SQL ==\nunknown(1,2,3)\n^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found ( at 14:15 expected ':', or data type"
       }
     },
     {
diff --git a/crates/sail-spark-connect/tests/gold_data/function/agg.json b/crates/sail-spark-connect/tests/gold_data/function/agg.json
index edd132214..ea0b53c21 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/agg.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/agg.json
@@ -1130,6 +1130,24 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT dimension_col, measure(measure_col)",
+        "result": [
+          "FROM test_metric_view",
+          "GROUP BY dimension_col;",
+          "dim_1, 100",
+          "dim_2, 200"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "analysis error: attribute ObjectName([Identifier(\"dimension_col\")]) is missing from the schema: cannot resolve attribute"
+      }
+    },
     {
       "input": {
         "query": "SELECT every(col) FROM VALUES (NULL), (true), (true) AS tab(col);",
@@ -1436,6 +1454,72 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_n_bigint(kll_merge_agg_bigint(sketch)) FROM (SELECT kll_sketch_agg_bigint(col) as sketch FROM VALUES (1), (2), (3) tab(col) UNION ALL SELECT kll_sketch_agg_bigint(col) as sketch FROM VALUES (4), (5), (6) tab(col)) t;",
+        "result": [
+          "6"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "kll_sketch_get_n_bigint(kll_merge_agg_bigint(sketch))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_n_double(kll_merge_agg_double(sketch)) FROM (SELECT kll_sketch_agg_double(col) as sketch FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)) tab(col) UNION ALL SELECT kll_sketch_agg_double(col) as sketch FROM VALUES (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)), (CAST(6.0 AS DOUBLE)) tab(col)) t;",
+        "result": [
+          "6"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "kll_sketch_get_n_double(kll_merge_agg_double(sketch))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_n_float(kll_merge_agg_float(sketch)) FROM (SELECT kll_sketch_agg_float(col) as sketch FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)) tab(col) UNION ALL SELECT kll_sketch_agg_float(col) as sketch FROM VALUES (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)), (CAST(6.0 AS FLOAT)) tab(col)) t;",
+        "result": [
+          "6"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "kll_sketch_get_n_float(kll_merge_agg_float(sketch))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_float"
+      }
+    },
     {
       "input": {
         "query": "SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);",
@@ -1810,6 +1894,32 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT max_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);",
+        "result": [
+          "[\"b\",\"c\"]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "max_by(x, y, 2)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "string",
+                "containsNull": true
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
     {
       "input": {
         "query": "SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col);",
@@ -1942,6 +2052,32 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT min_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);",
+        "result": [
+          "[\"a\",\"c\"]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "min_by(x, y, 2)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "string",
+                "containsNull": true
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
     {
       "input": {
         "query": "SELECT mode() WITHIN GROUP (ORDER BY col DESC) FROM VALUES (0), (10), (10), (20), (20) AS tab(col);",
@@ -2477,7 +2613,7 @@
         }
       },
       "output": {
-        "failure": "error in DataFusion: Error during planning: Failed to coerce arguments to satisfy a call to 'percentile_disc' function: coercion from Interval(YearMonth), Decimal128(2, 2) to the signature OneOf(Exact(Int8, Float64), Exact(Int16, Float64), Exact(Int32, Float64), Exact(Int64, Float64), Exact(UInt8, Float64), Exact(UInt16, Float64), Exact(UInt32, Float64), Exact(UInt64, Float64), Exact(Float16, Float64), Exact(Float32, Float64), Exact(Float64, Float64)) failed. No function matches the given name and argument types 'percentile_disc(Interval(YearMonth), Decimal128(2, 2))'. You might need to add explicit type casts.\n\tCandidate functions:\n\tpercentile_disc(Int8, Float64)\n\tpercentile_disc(Int16, Float64)\n\tpercentile_disc(Int32, Float64)\n\tpercentile_disc(Int64, Float64)\n\tpercentile_disc(UInt8, Float64)\n\tpercentile_disc(UInt16, Float64)\n\tpercentile_disc(UInt32, Float64)\n\tpercentile_disc(UInt64, Float64)\n\tpercentile_disc(Float16, Float64)\n\tpercentile_disc(Float32, Float64)\n\tpercentile_disc(Float64, Float64)"
+        "success": "ok"
       }
     },
     {
@@ -4240,6 +4376,138 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D), (3, 15.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D), (4, 12.0D) tab(key, summary));",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, 'sum')) FROM VALUES (1, 5.0D), (1, 1.0D), (2, 2.0D), (2, 3.0D), (3, 2.2D) tab(key, summary);",
+        "result": [
+          "3.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_union_agg_double(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D) tab(key, summary));",
+        "result": [
+          "3.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_union_agg_double(sketch, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 2), (3, 3), (4, 4) tab(key, summary));",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, 'sum')) FROM VALUES (1, 5), (1, 1), (2, 2), (2, 3), (3, 2) tab(key, summary);",
+        "result": [
+          "3.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 5), (2, 10) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 3), (3, 7) tab(key, summary));",
+        "result": [
+          "3.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
     {
       "input": {
         "query": "SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col);",
diff --git a/crates/sail-spark-connect/tests/gold_data/function/avro.json b/crates/sail-spark-connect/tests/gold_data/function/avro.json
new file mode 100644
index 000000000..fe899b757
--- /dev/null
+++ b/crates/sail-spark-connect/tests/gold_data/function/avro.json
@@ -0,0 +1,64 @@
+{
+  "tests": [
+    {
+      "input": {
+        "query": "SELECT from_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('u', NAMED_STRUCT('member0', member0, 'member1', member1)) AS s FROM VALUES (1, NULL), (NULL,  'a') tab(member0, member1));",
+        "result": [
+          "[false]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: from_avro"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT schema_of_avro('{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{\"name\": \"u\", \"type\": [\"int\", \"string\"]}]}', map());",
+        "result": [
+          "STRUCT NOT NULL>"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: schema_of_avro"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT to_avro(s) IS NULL FROM (SELECT NULL AS s);",
+        "result": [
+          "[true]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: to_avro"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT to_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}') IS NULL FROM (SELECT NULL AS s);",
+        "result": [
+          "[true]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: to_avro"
+      }
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/tests/gold_data/function/collection.json b/crates/sail-spark-connect/tests/gold_data/function/collection.json
index 2940e2767..547ea93d2 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/collection.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/collection.json
@@ -136,6 +136,28 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT hex(reverse(x'CAFE'));",
+        "result": [
+          "FECA"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "hex(reverse(X'CAFE'))",
+              "nullable": false,
+              "type": "string",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "error in DataFusion: Execution error: array_reverse does not support type 'Binary'."
+      }
+    },
     {
       "input": {
         "query": "SELECT reverse('Spark SQL');",
diff --git a/crates/sail-spark-connect/tests/gold_data/function/datetime.json b/crates/sail-spark-connect/tests/gold_data/function/datetime.json
index b01575b1e..061749060 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/datetime.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/datetime.json
@@ -2573,6 +2573,72 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT time_bucket(INTERVAL '1' HOUR, TIMESTAMP '2024-01-01 11:27:00');",
+        "result": [
+          "2024-01-01 11:00:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_bucket(INTERVAL '01' HOUR, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00')",
+              "nullable": false,
+              "type": "timestamp",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_bucket"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_bucket(INTERVAL '1' MONTH, TIMESTAMP '2024-07-20 14:30:00', TIMESTAMP '2024-06-15 09:00:00');",
+        "result": [
+          "2024-07-15 09:00:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_bucket(INTERVAL '1' MONTH, TIMESTAMP '2024-07-20 14:30:00', TIMESTAMP '2024-06-15 09:00:00')",
+              "nullable": false,
+              "type": "timestamp",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_bucket"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_bucket(INTERVAL '15' MINUTE, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00');",
+        "result": [
+          "2024-01-01 11:15:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_bucket(INTERVAL '15' MINUTE, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00')",
+              "nullable": false,
+              "type": "timestamp",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_bucket"
+      }
+    },
     {
       "input": {
         "query": "SELECT time_diff('HOUR', TIME'20:30:29', TIME'12:00:00');",
@@ -2639,6 +2705,534 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT time_from_micros(0);",
+        "result": [
+          "00:00:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_micros(0)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_micros(52200000000);",
+        "result": [
+          "14:30:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_micros(52200000000)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_micros(52200500000);",
+        "result": [
+          "14:30:00.5"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_micros(52200500000)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_micros(86399999999);",
+        "result": [
+          "23:59:59.999999"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_micros(86399999999)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_millis(0);",
+        "result": [
+          "00:00:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_millis(0)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_millis(52200000);",
+        "result": [
+          "14:30:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_millis(52200000)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_millis(52200500);",
+        "result": [
+          "14:30:00.5"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_millis(52200500)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_millis(86399999);",
+        "result": [
+          "23:59:59.999"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_millis(86399999)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_seconds(0);",
+        "result": [
+          "00:00:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_seconds(0)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_seconds(52200);",
+        "result": [
+          "14:30:00"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_seconds(52200)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_seconds(52200.5);",
+        "result": [
+          "14:30:00.5"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_seconds(52200.5)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_from_seconds(86399.999999);",
+        "result": [
+          "23:59:59.999999"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_from_seconds(86399.999999)",
+              "nullable": true,
+              "type": "time(6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_from_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_micros(TIME'00:00:00');",
+        "result": [
+          "0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_micros(TIME '00:00:00')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_micros(TIME'14:30:00');",
+        "result": [
+          "52200000000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_micros(TIME '14:30:00')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_micros(TIME'14:30:00.5');",
+        "result": [
+          "52200500000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_micros(TIME '14:30:00.5')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_micros(TIME'23:59:59.999999');",
+        "result": [
+          "86399999999"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_micros(TIME '23:59:59.999999')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_micros"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_millis(TIME'00:00:00');",
+        "result": [
+          "0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_millis(TIME '00:00:00')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_millis(TIME'14:30:00');",
+        "result": [
+          "52200000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_millis(TIME '14:30:00')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_millis(TIME'14:30:00.5');",
+        "result": [
+          "52200500"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_millis(TIME '14:30:00.5')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_millis(TIME'23:59:59.999');",
+        "result": [
+          "86399999"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_millis(TIME '23:59:59.999')",
+              "nullable": true,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_millis"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_seconds(TIME'00:00:00');",
+        "result": [
+          "0.000000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_seconds(TIME '00:00:00')",
+              "nullable": true,
+              "type": "decimal(14, 6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_seconds(TIME'14:30:00');",
+        "result": [
+          "52200.000000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_seconds(TIME '14:30:00')",
+              "nullable": true,
+              "type": "decimal(14, 6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_seconds(TIME'14:30:00.5');",
+        "result": [
+          "52200.500000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_seconds(TIME '14:30:00.5')",
+              "nullable": true,
+              "type": "decimal(14, 6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_seconds"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT time_to_seconds(TIME'23:59:59.999999');",
+        "result": [
+          "86399.999999"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "time_to_seconds(TIME '23:59:59.999999')",
+              "nullable": true,
+              "type": "decimal(14, 6)",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: time_to_seconds"
+      }
+    },
     {
       "input": {
         "query": "SELECT time_trunc('HOUR', TIME'09:32:05.359');",
@@ -2856,7 +3450,7 @@
         }
       },
       "output": {
-        "failure": "error in DataFusion: Execution error: Error parsing '12.10.05' as time. Tried formats: [\"HH.mm.ss\"]"
+        "success": "ok"
       }
     },
     {
@@ -3758,7 +4352,7 @@
         }
       },
       "output": {
-        "failure": "not supported: unknown function: try_to_time"
+        "success": "ok"
       }
     },
     {
@@ -3780,7 +4374,7 @@
         }
       },
       "output": {
-        "failure": "not supported: unknown function: try_to_time"
+        "success": "ok"
       }
     },
     {
@@ -3802,7 +4396,7 @@
         }
       },
       "output": {
-        "failure": "not supported: unknown function: try_to_time"
+        "success": "ok"
       }
     },
     {
diff --git a/crates/sail-spark-connect/tests/gold_data/function/json.json b/crates/sail-spark-connect/tests/gold_data/function/json.json
index 26bf3e1a3..156dce61b 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/json.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/json.json
@@ -542,6 +542,28 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT to_json(named_struct('b', 1, 'a', 2), map('sortKeys', 'true'));",
+        "result": [
+          "{\"a\":2,\"b\":1}"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "to_json(named_struct(b, 1, a, 2))",
+              "nullable": true,
+              "type": "string",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
     {
       "input": {
         "query": "SELECT to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));",
diff --git a/crates/sail-spark-connect/tests/gold_data/function/misc.json b/crates/sail-spark-connect/tests/gold_data/function/misc.json
index ac6fe1f9a..6ba83bc73 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/misc.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/misc.json
@@ -1,137 +1,5 @@
 {
   "tests": [
-    {
-      "input": {
-        "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(length(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(length(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(length(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_double"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(length(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_double"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(length(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_float"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(length(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_float"
-      }
-    },
     {
       "input": {
         "query": "SELECT aes_decrypt(unbase64('2NYmDCjgXTbbxGA3/SnJEfFC/JQ7olk2VQWReIAAFKo='), '1234567890abcdef', 'CBC');",
@@ -264,90 +132,6 @@
         "success": "ok"
       }
     },
-    {
-      "input": {
-        "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr);",
-        "result": [
-          "[{\"item\":0,\"count\":2},{\"item\":4,\"count\":2},{\"item\":1,\"count\":2},{\"item\":2,\"count\":1},{\"item\":3,\"count\":1}]"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 5)",
-              "nullable": false,
-              "type": {
-                "type": "array",
-                "elementType": {
-                  "type": "struct",
-                  "fields": [
-                    {
-                      "name": "item",
-                      "nullable": true,
-                      "type": "integer",
-                      "metadata": {}
-                    },
-                    {
-                      "name": "count",
-                      "nullable": false,
-                      "type": "long",
-                      "metadata": {}
-                    }
-                  ]
-                },
-                "containsNull": false
-              },
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: approx_top_k_accumulate"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr), 2) FROM VALUES 'a', 'b', 'c', 'c', 'c', 'c', 'd', 'd' tab(expr);",
-        "result": [
-          "[{\"item\":\"c\",\"count\":4},{\"item\":\"d\",\"count\":2}]"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 2)",
-              "nullable": false,
-              "type": {
-                "type": "array",
-                "elementType": {
-                  "type": "struct",
-                  "fields": [
-                    {
-                      "name": "item",
-                      "nullable": true,
-                      "type": "string",
-                      "metadata": {}
-                    },
-                    {
-                      "name": "count",
-                      "nullable": false,
-                      "type": "long",
-                      "metadata": {}
-                    }
-                  ]
-                },
-                "containsNull": false
-              },
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: approx_top_k_accumulate"
-      }
-    },
     {
       "input": {
         "query": "SELECT assert_true(0 < 1);",
@@ -658,15 +442,15 @@
     },
     {
       "input": {
-        "query": "SELECT current_schema();",
+        "query": "SELECT current_path();",
         "result": [
-          "default"
+          "system.builtin,system.session,spark_catalog.default"
         ],
         "schema": {
           "type": "struct",
           "fields": [
             {
-              "name": "current_schema()",
+              "name": "current_path()",
               "nullable": false,
               "type": "string",
               "metadata": {}
@@ -675,20 +459,20 @@
         }
       },
       "output": {
-        "success": "ok"
+        "failure": "not implemented: function: current_path"
       }
     },
     {
       "input": {
-        "query": "SELECT current_user();",
+        "query": "SELECT current_schema();",
         "result": [
-          "mockingjay"
+          "default"
         ],
         "schema": {
           "type": "struct",
           "fields": [
             {
-              "name": "current_user()",
+              "name": "current_schema()",
               "nullable": false,
               "type": "string",
               "metadata": {}
@@ -702,32 +486,24 @@
     },
     {
       "input": {
-        "query": "SELECT from_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('u', NAMED_STRUCT('member0', member0, 'member1', member1)) AS s FROM VALUES (1, NULL), (NULL,  'a') tab(member0, member1));",
-        "result": [
-          "[false]"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": []
-        }
-      },
-      "output": {
-        "failure": "not implemented: function: from_avro"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT from_protobuf(s, 'Person', '/path/to/descriptor.desc', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('name', name, 'id', id) AS s FROM VALUES ('John Doe', 1), (NULL,  2) tab(name, id));",
+        "query": "SELECT current_user();",
         "result": [
-          "[false]"
+          "mockingjay"
         ],
         "schema": {
           "type": "struct",
-          "fields": []
+          "fields": [
+            {
+              "name": "current_user()",
+              "nullable": false,
+              "type": "string",
+              "metadata": {}
+            }
+          ]
         }
       },
       "output": {
-        "failure": "not implemented: function: from_protobuf"
+        "success": "ok"
       }
     },
     {
@@ -774,50 +550,6 @@
         "success": "ok"
       }
     },
-    {
-      "input": {
-        "query": "SELECT hll_sketch_estimate(hll_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);",
-        "result": [
-          "3"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "hll_sketch_estimate(hll_sketch_agg(col, 12))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "success": "ok"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT hll_sketch_estimate(hll_union(hll_sketch_agg(col1), hll_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);",
-        "result": [
-          "6"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "hll_sketch_estimate(hll_union(hll_sketch_agg(col1, 12), hll_sketch_agg(col2, 12), false))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "success": "ok"
-      }
-    },
     {
       "input": {
         "query": "SELECT input_file_block_length();",
@@ -926,204 +658,6 @@
         "failure": "not implemented: function: java_method"
       }
     },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col);",
-        "result": [
-          "5"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_n_double(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
-        "result": [
-          "5"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "kll_sketch_get_n_double(kll_sketch_agg_double(col))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_double"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_n_float(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
-        "result": [
-          "5"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "kll_sketch_get_n_float(kll_sketch_agg_float(col))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_float"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_double"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_float"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_double"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
-        "result": [
-          "true"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "(kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3)",
-              "nullable": false,
-              "type": "boolean",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "failure": "not supported: unknown function: kll_sketch_agg_float"
-      }
-    },
     {
       "input": {
         "query": "SELECT monotonically_increasing_id();",
@@ -1212,21 +746,6 @@
         "failure": "not implemented: function: reflect"
       }
     },
-    {
-      "input": {
-        "query": "SELECT schema_of_avro('{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{\"name\": \"u\", \"type\": [\"int\", \"string\"]}]}', map());",
-        "result": [
-          "STRUCT NOT NULL>"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": []
-        }
-      },
-      "output": {
-        "failure": "not implemented: function: schema_of_avro"
-      }
-    },
     {
       "input": {
         "query": "SELECT session_user();",
@@ -1271,139 +790,6 @@
         "success": "ok"
       }
     },
-    {
-      "input": {
-        "query": "SELECT theta_sketch_estimate(theta_difference(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);",
-        "result": [
-          "2"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "theta_sketch_estimate(theta_difference(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "success": "ok"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);",
-        "result": [
-          "2"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "success": "ok"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT theta_sketch_estimate(theta_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);",
-        "result": [
-          "3"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "theta_sketch_estimate(theta_sketch_agg(col, 12))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "success": "ok"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT theta_sketch_estimate(theta_union(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);",
-        "result": [
-          "6"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "theta_sketch_estimate(theta_union(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12), 12))",
-              "nullable": false,
-              "type": "long",
-              "metadata": {}
-            }
-          ]
-        }
-      },
-      "output": {
-        "success": "ok"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT to_avro(s) IS NULL FROM (SELECT NULL AS s);",
-        "result": [
-          "[true]"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": []
-        }
-      },
-      "output": {
-        "failure": "not implemented: function: to_avro"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT to_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}') IS NULL FROM (SELECT NULL AS s);",
-        "result": [
-          "[true]"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": []
-        }
-      },
-      "output": {
-        "failure": "not implemented: function: to_avro"
-      }
-    },
-    {
-      "input": {
-        "query": "SELECT to_protobuf(s, 'Person', '/path/to/descriptor.desc', map('emitDefaultValues', 'true')) IS NULL FROM (SELECT NULL AS s);",
-        "result": [
-          "[true]"
-        ],
-        "schema": {
-          "type": "struct",
-          "fields": []
-        }
-      },
-      "output": {
-        "failure": "not implemented: function: to_protobuf"
-      }
-    },
     {
       "input": {
         "query": "SELECT try_aes_decrypt(unhex('----------468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM');",
diff --git a/crates/sail-spark-connect/tests/gold_data/function/protobuf.json b/crates/sail-spark-connect/tests/gold_data/function/protobuf.json
new file mode 100644
index 000000000..323b30682
--- /dev/null
+++ b/crates/sail-spark-connect/tests/gold_data/function/protobuf.json
@@ -0,0 +1,34 @@
+{
+  "tests": [
+    {
+      "input": {
+        "query": "SELECT from_protobuf(s, 'Person', '/path/to/descriptor.desc', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('name', name, 'id', id) AS s FROM VALUES ('John Doe', 1), (NULL,  2) tab(name, id));",
+        "result": [
+          "[false]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: from_protobuf"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT to_protobuf(s, 'Person', '/path/to/descriptor.desc', map('emitDefaultValues', 'true')) IS NULL FROM (SELECT NULL AS s);",
+        "result": [
+          "[true]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": []
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: to_protobuf"
+      }
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/tests/gold_data/function/sketch.json b/crates/sail-spark-connect/tests/gold_data/function/sketch.json
new file mode 100644
index 000000000..de1cd02bf
--- /dev/null
+++ b/crates/sail-spark-connect/tests/gold_data/function/sketch.json
@@ -0,0 +1,946 @@
+{
+  "tests": [
+    {
+      "input": {
+        "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(length(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(length(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(length(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(length(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(length(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_float"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(length(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_float"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr);",
+        "result": [
+          "[{\"item\":0,\"count\":2},{\"item\":4,\"count\":2},{\"item\":1,\"count\":2},{\"item\":2,\"count\":1},{\"item\":3,\"count\":1}]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 5)",
+              "nullable": false,
+              "type": {
+                "type": "array",
+                "elementType": {
+                  "type": "struct",
+                  "fields": [
+                    {
+                      "name": "item",
+                      "nullable": true,
+                      "type": "integer",
+                      "metadata": {}
+                    },
+                    {
+                      "name": "count",
+                      "nullable": false,
+                      "type": "long",
+                      "metadata": {}
+                    }
+                  ]
+                },
+                "containsNull": false
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: approx_top_k_accumulate"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr), 2) FROM VALUES 'a', 'b', 'c', 'c', 'c', 'c', 'd', 'd' tab(expr);",
+        "result": [
+          "[{\"item\":\"c\",\"count\":4},{\"item\":\"d\",\"count\":2}]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 2)",
+              "nullable": false,
+              "type": {
+                "type": "array",
+                "elementType": {
+                  "type": "struct",
+                  "fields": [
+                    {
+                      "name": "item",
+                      "nullable": true,
+                      "type": "string",
+                      "metadata": {}
+                    },
+                    {
+                      "name": "count",
+                      "nullable": false,
+                      "type": "long",
+                      "metadata": {}
+                    }
+                  ]
+                },
+                "containsNull": false
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: approx_top_k_accumulate"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT hll_sketch_estimate(hll_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);",
+        "result": [
+          "3"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "hll_sketch_estimate(hll_sketch_agg(col, 12))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT hll_sketch_estimate(hll_union(hll_sketch_agg(col1), hll_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);",
+        "result": [
+          "6"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "hll_sketch_estimate(hll_union(hll_sketch_agg(col1, 12), hll_sketch_agg(col2, 12), false))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col);",
+        "result": [
+          "5"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_n_double(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
+        "result": [
+          "5"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "kll_sketch_get_n_double(kll_sketch_agg_double(col))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_n_float(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
+        "result": [
+          "5"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "kll_sketch_get_n_float(kll_sketch_agg_float(col))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_float"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_float"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_bigint"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "(kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3)",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not supported: unknown function: kll_sketch_agg_float"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT theta_sketch_estimate(theta_difference(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);",
+        "result": [
+          "2"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "theta_sketch_estimate(theta_difference(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);",
+        "result": [
+          "2"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT theta_sketch_estimate(theta_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);",
+        "result": [
+          "3"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "theta_sketch_estimate(theta_sketch_agg(col, 12))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT theta_sketch_estimate(theta_union(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);",
+        "result": [
+          "6"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "theta_sketch_estimate(theta_union(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12), 12))",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "success": "ok"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_difference_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (5, 5.0D, 4, 4.0D), (1, 1.0D, 4, 4.0D), (2, 2.0D, 5, 5.0D), (3, 3.0D, 1, 1.0D) tab(col1, val1, col2, val2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_difference_double(tuple_sketch_agg_double(col1, val1, 12, sum), tuple_sketch_agg_double(col2, val2, 12, sum)))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_difference_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (5, 5.0D, 4), (1, 1.0D, 4), (2, 2.0D, 5), (3, 3.0D, 1) tab(col1, val1, col2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_difference_theta_double(tuple_sketch_agg_double(col1, val1, 12, sum), theta_sketch_agg(col2, 12)))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_intersection_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (1, 1.0D, 1, 4.0D), (2, 2.0D, 2, 5.0D), (3, 3.0D, 4, 6.0D) tab(col1, val1, col2, val2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_intersection_double(tuple_sketch_agg_double(col1, val1, 12, sum), tuple_sketch_agg_double(col2, val2, 12, sum), sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_intersection_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1.0D, 1), (2, 2.0D, 2), (3, 3.0D, 4) tab(col1, val1, col2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_intersection_theta_double(tuple_sketch_agg_double(col1, val1, 12, sum), theta_sketch_agg(col2, 12), sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (1, 2.0D), (2, 3.0D) tab(key, summary);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_union_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (1, 1.0D, 4, 4.0D), (2, 2.0D, 5, 5.0D), (3, 3.0D, 6, 6.0D) tab(col1, val1, col2, val2);",
+        "result": [
+          "6.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_union_double(tuple_sketch_agg_double(col1, val1, 12, sum), tuple_sketch_agg_double(col2, val2, 12, sum), 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_double(tuple_union_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1.0D, 4), (2, 2.0D, 5), (3, 3.0D, 6) tab(col1, val1, col2);",
+        "result": [
+          "6.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_double(tuple_union_theta_double(tuple_sketch_agg_double(col1, val1, 12, sum), theta_sketch_agg(col2, 12), 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_difference_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (5, 5, 4, 4), (1, 1, 4, 4), (2, 2, 5, 5), (3, 3, 1, 1) tab(col1, val1, col2, val2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_difference_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), tuple_sketch_agg_integer(col2, val2, 12, sum)))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_difference_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1) tab(col1, val1, col2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_difference_theta_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), theta_sketch_agg(col2, 12)))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_intersection_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (1, 1, 1, 4), (2, 2, 2, 5), (3, 3, 4, 6) tab(col1, val1, col2, val2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_intersection_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), tuple_sketch_agg_integer(col2, val2, 12, sum), sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_intersection_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1, 1), (2, 2, 2), (3, 3, 4) tab(col1, val1, col2);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_intersection_theta_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), theta_sketch_agg(col2, 12), sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (1, 2), (2, 3) tab(key, summary);",
+        "result": [
+          "2.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_union_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (1, 1, 4, 4), (2, 2, 5, 5), (3, 3, 6, 6) tab(col1, val1, col2, val2);",
+        "result": [
+          "6.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_union_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), tuple_sketch_agg_integer(col2, val2, 12, sum), 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_estimate_integer(tuple_union_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1, 4), (2, 2, 5), (3, 3, 6) tab(col1, val1, col2);",
+        "result": [
+          "6.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_estimate_integer(tuple_union_theta_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), theta_sketch_agg(col2, 12), 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_summary_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (1, 2.0D), (2, 3.0D) tab(key, summary);",
+        "result": [
+          "6.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_summary_double(tuple_sketch_agg_double(key, summary, 12, sum), sum)",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (1, 2), (2, 3) tab(key, summary);",
+        "result": [
+          "6"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, summary, 12, sum), sum)",
+              "nullable": false,
+              "type": "long",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_theta_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (2, 2.0D), (3, 3.0D) tab(key, summary);",
+        "result": [
+          "1.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_theta_double(tuple_sketch_agg_double(key, summary, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_double"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary);",
+        "result": [
+          "1.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, summary, 12, sum))",
+              "nullable": false,
+              "type": "double",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: tuple_sketch_agg_integer"
+      }
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/tests/gold_data/function/st.json b/crates/sail-spark-connect/tests/gold_data/function/st.json
index 843a5597b..f18e51971 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/st.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/st.json
@@ -10,7 +10,7 @@
           "type": "struct",
           "fields": [
             {
-              "name": "hex(st_asbinary(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040')))",
+              "name": "hex(st_asbinary(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040'), NDR))",
               "nullable": false,
               "type": "string",
               "metadata": {}
@@ -32,7 +32,7 @@
           "type": "struct",
           "fields": [
             {
-              "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040')))",
+              "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0), NDR))",
               "nullable": false,
               "type": "string",
               "metadata": {}
@@ -44,6 +44,50 @@
         "success": "ok"
       }
     },
+    {
+      "input": {
+        "query": "SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'), 'XDR'));",
+        "result": [
+          "00000000013FF00000000000004000000000000000"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0), XDR))",
+              "nullable": false,
+              "type": "string",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "error in DataFusion: Error during planning: one value expected: [ScalarFunction(ScalarFunction { func: ScalarUDF { inner: StGeomFromWKB { signature: Signature { type_signature: Exact([Binary]), volatility: Immutable, parameter_names: None } } }, args: [Literal(Binary(\"1,1,0,0,0,0,0,0,0,0,0,240,63,0,0,0,0,0,0,0,64\"), None)] }), Literal(Utf8(\"XDR\"), None)]"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326)));",
+        "result": [
+          "0101000000000000000000F03F0000000000000040"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326), NDR))",
+              "nullable": false,
+              "type": "string",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "error in DataFusion: Error during planning: one value expected: [Literal(Binary(\"1,1,0,0,0,0,0,0,0,0,0,240,63,0,0,0,0,0,0,0,64\"), None), Literal(Int32(4326), None)]"
+      }
+    },
     {
       "input": {
         "query": "SELECT st_srid(NULL);",
@@ -98,7 +142,7 @@
           "type": "struct",
           "fields": [
             {
-              "name": "st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'))",
+              "name": "st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0))",
               "nullable": false,
               "type": "integer",
               "metadata": {}
@@ -110,6 +154,28 @@
         "failure": "not implemented: function: st_srid"
       }
     },
+    {
+      "input": {
+        "query": "SELECT st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326));",
+        "result": [
+          "4326"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326))",
+              "nullable": false,
+              "type": "integer",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "error in DataFusion: Error during planning: one value expected: [Literal(Binary(\"1,1,0,0,0,0,0,0,0,0,0,240,63,0,0,0,0,0,0,0,64\"), None), Literal(Int32(4326), None)]"
+      }
+    },
     {
       "input": {
         "query": "SELECT st_srid(st_setsrid(ST_GeogFromWKB(X'0101000000000000000000F03F0000000000000040'), 4326));",
@@ -142,7 +208,7 @@
           "type": "struct",
           "fields": [
             {
-              "name": "st_srid(st_setsrid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'), 3857))",
+              "name": "st_srid(st_setsrid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0), 3857))",
               "nullable": false,
               "type": "integer",
               "metadata": {}
diff --git a/crates/sail-spark-connect/tests/gold_data/function/variant.json b/crates/sail-spark-connect/tests/gold_data/function/variant.json
index e681b7457..b17f5a312 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/variant.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/variant.json
@@ -1,5 +1,49 @@
 {
   "tests": [
+    {
+      "input": {
+        "query": "SELECT is_valid_variant(parse_json('[{\"b\":true,\"a\":0}]'));",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "is_valid_variant(parse_json([{\"b\":true,\"a\":0}]))",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: is_valid_variant"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT is_valid_variant(parse_json('null'));",
+        "result": [
+          "true"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "is_valid_variant(parse_json(null))",
+              "nullable": false,
+              "type": "boolean",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: is_valid_variant"
+      }
+    },
     {
       "input": {
         "query": "SELECT is_variant_null(parse_json('\"null\"'));",
diff --git a/crates/sail-spark-connect/tests/gold_data/function/vector.json b/crates/sail-spark-connect/tests/gold_data/function/vector.json
new file mode 100644
index 000000000..16b4c6a19
--- /dev/null
+++ b/crates/sail-spark-connect/tests/gold_data/function/vector.json
@@ -0,0 +1,266 @@
+{
+  "tests": [
+    {
+      "input": {
+        "query": "SELECT vector_avg(col) FROM VALUES (array(1.0F, 2.0F)), (array(3.0F, 4.0F)) AS tab(col);",
+        "result": [
+          "[2.0,3.0]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_avg(col)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "float",
+                "containsNull": false
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_avg"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_cosine_similarity(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));",
+        "result": [
+          "0.9746319"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_cosine_similarity(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0))",
+              "nullable": true,
+              "type": "float",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_cosine_similarity"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_inner_product(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));",
+        "result": [
+          "32.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_inner_product(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0))",
+              "nullable": true,
+              "type": "float",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_inner_product"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_l2_distance(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));",
+        "result": [
+          "5.196152"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_l2_distance(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0))",
+              "nullable": true,
+              "type": "float",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_l2_distance"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_norm(array(3.0F, 4.0F), 1.0F);",
+        "result": [
+          "7.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_norm(array(3.0, 4.0), 1.0)",
+              "nullable": true,
+              "type": "float",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_norm"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_norm(array(3.0F, 4.0F), 2.0F);",
+        "result": [
+          "5.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_norm(array(3.0, 4.0), 2.0)",
+              "nullable": true,
+              "type": "float",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_norm"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_norm(array(3.0F, 4.0F), float('inf'));",
+        "result": [
+          "4.0"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_norm(array(3.0, 4.0), inf)",
+              "nullable": true,
+              "type": "float",
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_norm"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_normalize(array(3.0F, 4.0F), 1.0F);",
+        "result": [
+          "[0.42857143,0.5714286]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_normalize(array(3.0, 4.0), 1.0)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "float",
+                "containsNull": true
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_normalize"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_normalize(array(3.0F, 4.0F), 2.0F);",
+        "result": [
+          "[0.6,0.8]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_normalize(array(3.0, 4.0), 2.0)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "float",
+                "containsNull": true
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_normalize"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_normalize(array(3.0F, 4.0F), float('inf'));",
+        "result": [
+          "[0.75,1.0]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_normalize(array(3.0, 4.0), inf)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "float",
+                "containsNull": true
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_normalize"
+      }
+    },
+    {
+      "input": {
+        "query": "SELECT vector_sum(col) FROM VALUES (array(1.0F, 2.0F)), (array(3.0F, 4.0F)) AS tab(col);",
+        "result": [
+          "[4.0,6.0]"
+        ],
+        "schema": {
+          "type": "struct",
+          "fields": [
+            {
+              "name": "vector_sum(col)",
+              "nullable": true,
+              "type": {
+                "type": "array",
+                "elementType": "float",
+                "containsNull": false
+              },
+              "metadata": {}
+            }
+          ]
+        }
+      },
+      "output": {
+        "failure": "not implemented: function: vector_sum"
+      }
+    }
+  ]
+}
diff --git a/crates/sail-spark-connect/tests/gold_data/function/xml.json b/crates/sail-spark-connect/tests/gold_data/function/xml.json
index a36a314b4..e9b690ef8 100644
--- a/crates/sail-spark-connect/tests/gold_data/function/xml.json
+++ b/crates/sail-spark-connect/tests/gold_data/function/xml.json
@@ -35,7 +35,7 @@
         }
       },
       "output": {
-        "failure": "not implemented: function: from_xml"
+        "success": "ok"
       }
     },
     {
@@ -93,7 +93,7 @@
         }
       },
       "output": {
-        "failure": "not implemented: function: from_xml"
+        "success": "ok"
       }
     },
     {
@@ -125,7 +125,7 @@
         }
       },
       "output": {
-        "failure": "not implemented: function: from_xml"
+        "success": "ok"
       }
     },
     {
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json
index f6f37db52..9e396f0a0 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json
@@ -67,76 +67,6 @@
           }
         }
       }
-    },
-    {
-      "input": "DESCRIBE TABLE EXTENDED a.b",
-      "output": {
-        "success": {
-          "command": {
-            "describeTable": {
-              "table": [
-                "a",
-                "b"
-              ],
-              "extended": true,
-              "partition": [],
-              "column": null
-            },
-            "planId": null
-          }
-        }
-      }
-    },
-    {
-      "input": "DESCRIBE EXTENDED a.b",
-      "output": {
-        "success": {
-          "command": {
-            "describeTable": {
-              "table": [
-                "a",
-                "b"
-              ],
-              "extended": true,
-              "partition": [],
-              "column": null
-            },
-            "planId": null
-          }
-        }
-      }
-    },
-    {
-      "input": "DESCRIBE QUERY SELECT 1",
-      "output": {
-        "success": {
-          "command": {
-            "describeQuery": {
-              "query": {
-                "project": {
-                  "input": {
-                    "empty": {
-                      "produceOneRow": true
-                    },
-                    "planId": null
-                  },
-                  "expressions": [
-                    {
-                      "literal": {
-                        "int32": {
-                          "value": 1
-                        }
-                      }
-                    }
-                  ]
-                },
-                "planId": null
-              }
-            },
-            "planId": null
-          }
-        }
-      }
     }
   ]
 }
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json
index e9b150110..b1f435f7f 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json
@@ -811,6 +811,12 @@
         }
       }
     },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl\nREPLACE WHERE a > 5\n(SELECT * FROM source) AS s",
+      "output": {
+        "failure": "invalid argument: found AS at 75:77 expected 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "INSERT INTO testcat.ns1.ns2.tbl (a, b) SELECT * FROM source",
       "output": {
@@ -874,6 +880,74 @@
         }
       }
     },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t\nREPLACE ON (t.a = s.b)\n(SELECT * FROM source) AS s",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t\nREPLACE USING (col1, col2)\n(SELECT * FROM source) AS s",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE ON t.col1 = col2 SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE USING (col1, col2) SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE WHERE a > 5 SELECT * FROM source",
+      "exception": "\n[INSERT_REPLACE_WHERE_TABLE_ALIAS_NOT_ALLOWED] Table alias is not allowed with INSERT INTO ... REPLACE WHERE because the WHERE condition is evaluated against the target table directly.\nUse INSERT INTO ... REPLACE ON if you need to reference the target table via an alias. SQLSTATE: 42000\n== SQL (line 1, position 1) ==\nINSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE WHERE a > 5 SELECT * FROM source\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE ON t.a = s.a AND t.b = s.b (SELECT * FROM source) AS s",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE ON t.col1 = col2 SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE USING (col1, col2) SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE USING (id) SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE WHERE a > 5 SELECT * FROM source",
+      "exception": "\n[INSERT_REPLACE_WHERE_TABLE_ALIAS_NOT_ALLOWED] Table alias is not allowed with INSERT INTO ... REPLACE WHERE because the WHERE condition is evaluated against the target table directly.\nUse INSERT INTO ... REPLACE ON if you need to reference the target table via an alias. SQLSTATE: 42000\n== SQL (line 1, position 1) ==\nINSERT INTO testcat.ns1.ns2.tbl AS t REPLACE WHERE a > 5 SELECT * FROM source\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl BY NAME REPLACE WHERE a > 5 SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found REPLACE at 40:47 expected query"
+      }
+    },
     {
       "input": "INSERT INTO testcat.ns1.ns2.tbl BY NAME SELECT * FROM source",
       "output": {
@@ -933,6 +1007,105 @@
         }
       }
     },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl REPLACE ON col1 = col2 SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found ON at 40:42 expected 'WHERE'"
+      }
+    },
+    {
+      "input": "INSERT INTO testcat.ns1.ns2.tbl REPLACE WHERE a > 5 SELECT * FROM source",
+      "output": {
+        "success": {
+          "command": {
+            "insertInto": {
+              "input": {
+                "project": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "source"
+                        ],
+                        "temporal": null,
+                        "sample": null,
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "expressions": [
+                    {
+                      "unresolvedStar": {
+                        "target": null,
+                        "planId": null,
+                        "wildcardOptions": {
+                          "ilikePattern": null,
+                          "excludeColumns": null,
+                          "exceptColumns": null,
+                          "replaceColumns": null,
+                          "renameColumns": null
+                        }
+                      }
+                    }
+                  ]
+                },
+                "planId": null
+              },
+              "table": [
+                "testcat",
+                "ns1",
+                "ns2",
+                "tbl"
+              ],
+              "mode": {
+                "replace": {
+                  "condition": {
+                    "expr": {
+                      "unresolvedFunction": {
+                        "functionName": [
+                          ">"
+                        ],
+                        "arguments": [
+                          {
+                            "unresolvedAttribute": {
+                              "name": [
+                                "a"
+                              ],
+                              "planId": null,
+                              "isMetadataColumn": false
+                            }
+                          },
+                          {
+                            "literal": {
+                              "int32": {
+                                "value": 5
+                              }
+                            }
+                          }
+                        ],
+                        "namedArguments": [],
+                        "isDistinct": false,
+                        "isUserDefinedFunction": false,
+                        "isInternal": null,
+                        "ignoreNulls": null,
+                        "filter": null,
+                        "orderBy": null
+                      }
+                    },
+                    "source": "a > 5 "
+                  }
+                }
+              },
+              "partition": [],
+              "ifNotExists": false
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "INSERT INTO testcat.ns1.ns2.tbl SELECT * FROM source",
       "output": {
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json
index b4520a1ac..30c0f2599 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json
@@ -2576,6 +2576,13 @@
         }
       }
     },
+    {
+      "input": "MERGE INTO t1 USING t2 ON t1.id = t2.id\nWHEN NOT MATCHED THEN INSERT (col1, col2) VALUES (1)",
+      "exception": "\n[MERGE_INSERT_VALUE_COUNT_MISMATCH] The number of provided values (1) must match the number of expected columns (2) in the INSERT clause of MERGE. SQLSTATE: 21S01\n== SQL (line 1, position 1) ==\nMERGE INTO t1 USING t2 ON t1.id = t2.id\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nWHEN NOT MATCHED THEN INSERT (col1, col2) VALUES (1)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: INSERT action has 2 columns but 1 expressions"
+      }
+    },
     {
       "input": "MERGE INTO testcat1.ns1.ns2.tbl AS target\nUSING testcat2.ns1.ns2.tbl AS source\nON target.col1 = source.col1",
       "exception": "\n[MERGE_WITHOUT_WHEN] There must be at least one WHEN clause in a MERGE statement. SQLSTATE: 42601\n== SQL (line 1, position 1) ==\nMERGE INTO testcat1.ns1.ns2.tbl AS target\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nUSING testcat2.ns1.ns2.tbl AS source\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nON target.col1 = source.col1\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json
index 3f08fb22f..4e7f30cc5 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json
@@ -150,6 +150,66 @@
         }
       }
     },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl (a, b)SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE ON t.col1 = col2 SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl AS t REPLACE ON t.col1 = col2 SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl BY NAME REPLACE USING (col1) SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl BY NAME SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl REPLACE USING (col1) SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION OVERWRITE testcat.ns1.ns2.tbl (a, b)SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION OVERWRITE testcat.ns1.ns2.tbl BY NAME SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
+    {
+      "input": "INSERT WITH SCHEMA EVOLUTION OVERWRITE testcat.ns1.ns2.tbl SELECT * FROM source",
+      "output": {
+        "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'"
+      }
+    },
     {
       "input": "REFRESH FUNCTION a.b.c",
       "output": {
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json b/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json
index 73e4a594e..669ba5c01 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json
@@ -44,7 +44,7 @@
     },
     {
       "input": "SET CATALOG test-test",
-      "exception": "\n[INVALID_IDENTIFIER] The unquoted identifier test-test is invalid and must be back quoted as: `test-test`.\nUnquoted identifiers can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_').\nUnquoted identifiers must also not start with a digit.\nDifferent data sources and meta stores may impose additional restrictions on valid identifiers. SQLSTATE: 42602 (line 1, pos 16)\n\n== SQL ==\nSET CATALOG test-test\n----------------^^^\n",
+      "exception": "\n[INVALID_SQL_SYNTAX.UNSUPPORTED_SQL_STATEMENT] Invalid SQL syntax: Unsupported SQL statement: SET CATALOG test-test. SQLSTATE: 42000\n== SQL (line 1, position 1) ==\nSET CATALOG test-test\n^^^^^^^^^^^^^^^^^^^^^\n",
       "output": {
         "failure": "invalid argument: found - at 16:17 expected ';', or end of input"
       }
@@ -53,7 +53,7 @@
       "input": "SHOW FUNCTIONS LIKE test-func",
       "exception": "\n[INVALID_IDENTIFIER] The unquoted identifier test-func is invalid and must be back quoted as: `test-func`.\nUnquoted identifiers can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_').\nUnquoted identifiers must also not start with a digit.\nDifferent data sources and meta stores may impose additional restrictions on valid identifiers. SQLSTATE: 42602 (line 1, pos 24)\n\n== SQL ==\nSHOW FUNCTIONS LIKE test-func\n------------------------^^^\n",
       "output": {
-        "failure": "invalid argument: found LIKE at 15:19 expected ';', or end of input"
+        "failure": "invalid argument: found - at 24:25 expected '.', ';', or end of input"
       }
     },
     {
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json b/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json
index 19f6bded0..ca80fdfd3 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json
@@ -126,6 +126,12 @@
         }
       }
     },
+    {
+      "input": "SELECT * FROM t1 CHANGES FROM VERSION 1 TO VERSION 5 AS a JOIN t2 CHANGES FROM VERSION 1 TO VERSION 5 AS b ON a.id = b.id",
+      "output": {
+        "failure": "invalid argument: found FROM at 25:29 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from a join b join c right join d",
       "output": {
@@ -1476,6 +1482,13 @@
         }
       }
     },
+    {
+      "input": "select * from t cross join u approx nearest 1 by similarity t.a",
+      "exception": "\n[NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE] Invalid nearest-by join. Unsupported nearest-by join type CROSS. Supported types: 'INNER', 'LEFT OUTER'. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t cross join u approx nearest 1 by similarity t.a\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found nearest at 36:43 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t full join u as uu on a = b",
       "output": {
@@ -1649,6 +1662,13 @@
         }
       }
     },
+    {
+      "input": "select * from t full outer join u approx nearest 1 by similarity t.a",
+      "exception": "\n[NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE] Invalid nearest-by join. Unsupported nearest-by join type FULL OUTER. Supported types: 'INNER', 'LEFT OUTER'. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t full outer join u approx nearest 1 by similarity t.a\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found nearest at 41:48 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t full outer join u as uu on a = b",
       "output": {
@@ -2016,6 +2036,12 @@
         }
       }
     },
+    {
+      "input": "select * from t inner join u exact nearest 3 by distance t.a - u.a",
+      "output": {
+        "failure": "invalid argument: found nearest at 35:42 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t inner join u using(a, b)",
       "output": {
@@ -2173,6 +2199,13 @@
         }
       }
     },
+    {
+      "input": "select * from t join lateral (select * from u) uu approx nearest 1 by similarity 1",
+      "exception": "\n[UNSUPPORTED_FEATURE.LATERAL_JOIN_NEAREST_BY] The feature is not supported: LATERAL correlation with NEAREST BY clause. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t join lateral (select * from u) uu approx nearest 1 by similarity 1\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found approx at 50:56 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t join lateral (select * from u) uu on true",
       "output": {
@@ -2268,6 +2301,33 @@
         }
       }
     },
+    {
+      "input": "select * from t join u approx nearest 0 by similarity t.a",
+      "exception": "\n[NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE] Invalid nearest-by join. The number of results 0 must be between 1 and 100000. Update the literal in `APPROX NEAREST 0 BY ...` (or `EXACT NEAREST 0 BY ...`) to fall within that range. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t join u approx nearest 0 by similarity t.a\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "select * from t join u approx nearest 100001 by distance t.a",
+      "exception": "\n[NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE] Invalid nearest-by join. The number of results 100001 must be between 1 and 100000. Update the literal in `APPROX NEAREST 100001 BY ...` (or `EXACT NEAREST 100001 BY ...`) to fall within that range. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t join u approx nearest 100001 by distance t.a\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "select * from t join u approx nearest 5 by similarity t.a + u.a",
+      "output": {
+        "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "select * from t join u approx nearest 99999999999999999999 by distance t.a",
+      "exception": "\n[NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE] Invalid nearest-by join. The number of results 99999999999999999999 must be between 1 and 100000. Update the literal in `APPROX NEAREST 99999999999999999999 BY ...` (or `EXACT NEAREST 99999999999999999999 BY ...`) to fall within that range. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t join u approx nearest 99999999999999999999 by distance t.a\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t join u as uu on a = b",
       "output": {
@@ -2965,6 +3025,12 @@
         }
       }
     },
+    {
+      "input": "select * from t left outer join u approx nearest by similarity t.a + u.a",
+      "output": {
+        "failure": "invalid argument: found nearest at 41:48 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t left outer join u as uu on a = b",
       "output": {
@@ -3484,6 +3550,13 @@
         }
       }
     },
+    {
+      "input": "select * from t right outer join u approx nearest 1 by similarity t.a",
+      "exception": "\n[NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE] Invalid nearest-by join. Unsupported nearest-by join type RIGHT OUTER. Supported types: 'INNER', 'LEFT OUTER'. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t right outer join u approx nearest 1 by similarity t.a\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found nearest at 42:49 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "select * from t right outer join u as uu on a = b",
       "output": {
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json b/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json
index 186bce4ba..7e6d5b8f1 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json
@@ -115,6 +115,18 @@
         "failure": "invalid argument: found INTO at 22:26 expected '.', '(', 'FOR', 'SYSTEM_VERSION', 'VERSION', 'SYSTEM_TIME', 'TIMESTAMP', 'TABLESAMPLE', 'PIVOT', 'UNPIVOT', 'AS', identifier, 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', or ')'"
       }
     },
+    {
+      "input": "SELECT * FROM (SELECT * FROM t CHANGES FROM VERSION 1 TO VERSION 5) sub",
+      "output": {
+        "failure": "invalid argument: found FROM at 39:43 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', or ')'"
+      }
+    },
+    {
+      "input": "SELECT * FROM (SELECT * FROM t TABLESAMPLE SYSTEM (50 PERCENT)) sub",
+      "output": {
+        "failure": "invalid argument: found SYSTEM at 43:49 expected '('"
+      }
+    },
     {
       "input": "SELECT * FROM (SELECT a AS x, b AS y FROM t) t(col1, col2)",
       "output": {
@@ -209,7 +221,7 @@
     },
     {
       "input": "SELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))",
-      "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'IN'. SQLSTATE: 42601 (line 1, pos 25)\n\n== SQL ==\nSELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))\n-------------------------^^^\n",
+      "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'INTO'. SQLSTATE: 42601 (line 1, pos 36)\n\n== SQL ==\nSELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))\n------------------------------------^^^\n",
       "output": {
         "failure": "invalid argument: found INTO at 36:40 expected '->', '.', '(', '[', '::', 'ESCAPE', 'IS', 'NOT', 'IN', '*', '/', '%', 'DIV', '+', '-', '||', '>>>', '>>', '<<', '&', '^', '|', '!=', '!>', '!<', '==', '=', '>=', '>', '<=>', '<=', '<>', '<', 'BETWEEN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'SIMILAR', 'AND', 'OR', ',', or ')'"
       }
@@ -309,6 +321,121 @@
         }
       }
     },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM SYSTEM_TIME '2026-01-01' TO SYSTEM_TIME '2026-02-01'",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM SYSTEM_VERSION 10 TO SYSTEM_VERSION 20",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01'",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01' EXCLUSIVE TO TIMESTAMP '2026-02-01' EXCLUSIVE",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01' INCLUSIVE TO TIMESTAMP '2026-02-01' INCLUSIVE",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01' TO TIMESTAMP '2026-02-01'",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION '5765898212649545898' TO VERSION '8439568982649545102'",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 EXCLUSIVE TO VERSION 20",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 EXCLUSIVE TO VERSION 20 EXCLUSIVE",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 INCLUSIVE TO VERSION 20 INCLUSIVE",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 EXCLUSIVE",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (computeUpdates = 'true')",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'DROPCARRYOVERS')",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'invalid')",
+      "exception": "\n[INVALID_CDC_OPTION.INVALID_DEDUPLICATION_MODE] Invalid Change Data Capture (CDC) option. Invalid `deduplicationMode`: 'invalid'. Expected one of: none, dropCarryovers, netChanges. SQLSTATE: 42K03\n== SQL ==\nSELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'invalid')",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'netChanges')",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'none')",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'none', computeUpdates = 'true')",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 5",
+      "output": {
+        "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "SELECT * FROM a.b.c FOR SYSTEM_TIME AS OF '2019-01-29 00:37:58'",
       "output": {
@@ -1557,6 +1684,85 @@
         }
       }
     },
+    {
+      "input": "SELECT * FROM bernoulli TABLESAMPLE BERNOULLI (10 PERCENT) AS x",
+      "output": {
+        "failure": "invalid argument: found BERNOULLI at 36:45 expected '('"
+      }
+    },
+    {
+      "input": "SELECT * FROM bernoulli TABLESAMPLE(10 PERCENT) bernoulli",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "bernoulli"
+                        ],
+                        "temporal": null,
+                        "sample": {
+                          "method": {
+                            "percent": {
+                              "value": {
+                                "literal": {
+                                  "int32": {
+                                    "value": 10
+                                  }
+                                }
+                              }
+                            }
+                          },
+                          "seed": null
+                        },
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "name": "bernoulli",
+                  "columns": []
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "SELECT * FROM my_table CHANGES FROM VERSION 1",
+      "output": {
+        "failure": "invalid argument: found FROM at 31:35 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM mydb.system TABLESAMPLE SYSTEM (10 PERCENT) AS x",
+      "output": {
+        "failure": "invalid argument: found SYSTEM at 38:44 expected '('"
+      }
+    },
     {
       "input": "SELECT * FROM parquet_t0 TABLESAMPLE(300M) s",
       "exception": "\nTABLESAMPLE(byteLengthLiteral) is not supported.\n== SQL (line 1, position 26) ==\nSELECT * FROM parquet_t0 TABLESAMPLE(300M) s\n                         ^^^^^^^^^^^^^^^^^\n",
@@ -1686,7 +1892,19 @@
       }
     },
     {
-      "input": "SELECT * FROM testData AS t(col1, col2)",
+      "input": "SELECT * FROM system TABLESAMPLE BERNOULLI (10 PERCENT) AS x",
+      "output": {
+        "failure": "invalid argument: found BERNOULLI at 33:42 expected '('"
+      }
+    },
+    {
+      "input": "SELECT * FROM system TABLESAMPLE SYSTEM (10 PERCENT) AS x",
+      "output": {
+        "failure": "invalid argument: found SYSTEM at 33:39 expected '('"
+      }
+    },
+    {
+      "input": "SELECT * FROM system TABLESAMPLE(10 PERCENT) AS x",
       "output": {
         "success": {
           "query": {
@@ -1697,21 +1915,31 @@
                     "read": {
                       "namedTable": {
                         "name": [
-                          "testData"
+                          "system"
                         ],
                         "temporal": null,
-                        "sample": null,
+                        "sample": {
+                          "method": {
+                            "percent": {
+                              "value": {
+                                "literal": {
+                                  "int32": {
+                                    "value": 10
+                                  }
+                                }
+                              }
+                            }
+                          },
+                          "seed": null
+                        },
                         "options": []
                       },
                       "isStreaming": false
                     },
                     "planId": null
                   },
-                  "name": "t",
-                  "columns": [
-                    "col1",
-                    "col2"
-                  ]
+                  "name": "x",
+                  "columns": []
                 },
                 "planId": null
               },
@@ -1737,24 +1965,42 @@
       }
     },
     {
-      "input": "SELECT * FROM testcat.db.tab",
+      "input": "SELECT * FROM system TABLESAMPLE(10 PERCENT) system",
       "output": {
         "success": {
           "query": {
             "project": {
               "input": {
-                "read": {
-                  "namedTable": {
-                    "name": [
-                      "testcat",
-                      "db",
-                      "tab"
-                    ],
-                    "temporal": null,
-                    "sample": null,
-                    "options": []
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "system"
+                        ],
+                        "temporal": null,
+                        "sample": {
+                          "method": {
+                            "percent": {
+                              "value": {
+                                "literal": {
+                                  "int32": {
+                                    "value": 10
+                                  }
+                                }
+                              }
+                            }
+                          },
+                          "seed": null
+                        },
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
                   },
-                  "isStreaming": false
+                  "name": "system",
+                  "columns": []
                 },
                 "planId": null
               },
@@ -1780,22 +2026,233 @@
       }
     },
     {
-      "input": "SELECT 42",
+      "input": "SELECT * FROM t CHANGES FROM TIMESTAMP '2026-01-01' TO TIMESTAMP (SELECT MAX(ts) FROM other_table)",
+      "exception": "\n[INVALID_CDC_OPTION.INVALID_TIMESTAMP_EXPR] Invalid Change Data Capture (CDC) option. The timestamp expression for CDC queries must not contain subqueries. SQLSTATE: 42K03\n== SQL ==\nSELECT * FROM t CHANGES FROM TIMESTAMP '2026-01-01' TO TIMESTAMP (SELECT MAX(ts) FROM other_table)",
+      "output": {
+        "failure": "invalid argument: found FROM at 24:28 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM t CHANGES FROM TIMESTAMP (SELECT MAX(ts) FROM other_table)",
+      "exception": "\n[INVALID_CDC_OPTION.INVALID_TIMESTAMP_EXPR] Invalid Change Data Capture (CDC) option. The timestamp expression for CDC queries must not contain subqueries. SQLSTATE: 42K03\n== SQL ==\nSELECT * FROM t CHANGES FROM TIMESTAMP (SELECT MAX(ts) FROM other_table)",
+      "output": {
+        "failure": "invalid argument: found FROM at 24:28 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM t CHANGES FROM TIMESTAMP col_name",
+      "exception": "\nInvalid time travel spec: timestamp expression cannot refer to any columns.\n== SQL (line 1, position 15) ==\nSELECT * FROM t CHANGES FROM TIMESTAMP col_name\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found FROM at 24:28 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
+    {
+      "input": "SELECT * FROM t bernoulli",
       "output": {
         "success": {
           "query": {
             "project": {
               "input": {
-                "empty": {
-                  "produceOneRow": true
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "t"
+                        ],
+                        "temporal": null,
+                        "sample": null,
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "name": "bernoulli",
+                  "columns": []
                 },
                 "planId": null
               },
               "expressions": [
                 {
-                  "literal": {
-                    "int32": {
-                      "value": 42
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "SELECT * FROM t system",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "t"
+                        ],
+                        "temporal": null,
+                        "sample": null,
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "name": "system",
+                  "columns": []
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "SELECT * FROM testData AS t(col1, col2)",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "testData"
+                        ],
+                        "temporal": null,
+                        "sample": null,
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "name": "t",
+                  "columns": [
+                    "col1",
+                    "col2"
+                  ]
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "SELECT * FROM testcat.db.tab",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "testcat",
+                      "db",
+                      "tab"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "SELECT 42",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "empty": {
+                  "produceOneRow": true
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "literal": {
+                    "int32": {
+                      "value": 42
                     }
                   }
                 }
@@ -2427,6 +2884,43 @@
         }
       }
     },
+    {
+      "input": "SELECT bernoulli FROM t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "bernoulli"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "SELECT interval FROM VALUES ('abc') AS tbl(interval);",
       "output": {
@@ -2473,6 +2967,49 @@
         }
       }
     },
+    {
+      "input": "SELECT system FROM t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "system"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "SELECT t.* FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 AS t",
+      "output": {
+        "failure": "invalid argument: found FROM at 30:34 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input"
+      }
+    },
     {
       "input": "SELECT t.col1, t.col2 FROM (SELECT a AS x, b AS y FROM t) t(col1, col2)",
       "output": {
@@ -3008,20 +3545,60 @@
       }
     },
     {
-      "input": "select * from default.range(2)",
-      "exception": "\n[INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME] Invalid SQL syntax: Table valued function cannot specify database name: `default`.`range`. SQLSTATE: 42000\n== SQL (line 1, position 15) ==\nselect * from default.range(2)\n              ^^^^^^^^^^^^^^^^\n",
+      "input": "select * from approx",
       "output": {
         "success": {
           "query": {
             "project": {
               "input": {
                 "read": {
-                  "udtf": {
+                  "namedTable": {
                     "name": [
-                      "default",
-                      "range"
+                      "approx"
                     ],
-                    "arguments": [
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "select * from default.range(2)",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "udtf": {
+                    "name": [
+                      "default",
+                      "range"
+                    ],
+                    "arguments": [
                       {
                         "literal": {
                           "int32": {
@@ -3058,6 +3635,88 @@
         }
       }
     },
+    {
+      "input": "select * from distance",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "distance"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "select * from exact",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "exact"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "select * from my_tvf(2, arg1 => 'value1', arg2 => true)",
       "output": {
@@ -3604,6 +4263,47 @@
         }
       }
     },
+    {
+      "input": "select * from nearest",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "nearest"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "select * from range(2)",
       "output": {
@@ -3653,9 +4353,49 @@
         }
       }
     },
+    {
+      "input": "select * from similarity",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "similarity"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "select * from spark_catalog.default.range(2)",
-      "exception": "\n[INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME] Invalid SQL syntax: Table valued function cannot specify database name: `spark_catalog`.`default`.`range`. SQLSTATE: 42000\n== SQL (line 1, position 15) ==\nselect * from spark_catalog.default.range(2)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "output": {
         "success": {
           "query": {
@@ -3835,6 +4575,24 @@
         }
       }
     },
+    {
+      "input": "select * from t TABLESAMPLE BERNOULLI (43 PERCENT) as x",
+      "output": {
+        "failure": "invalid argument: found BERNOULLI at 28:37 expected '('"
+      }
+    },
+    {
+      "input": "select * from t TABLESAMPLE SYSTEM (43 PERCENT) as x",
+      "output": {
+        "failure": "invalid argument: found SYSTEM at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t TabLeSaMpLe SyStEm (43 PeRcEnT) as x",
+      "output": {
+        "failure": "invalid argument: found SyStEm at 28:34 expected '('"
+      }
+    },
     {
       "input": "select * from t as tt , u",
       "output": {
@@ -4187,6 +4945,98 @@
         }
       }
     },
+    {
+      "input": "select * from t tablesample bernoulli (43 percent) as x",
+      "output": {
+        "failure": "invalid argument: found bernoulli at 28:37 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample bernoulli (43 percent) repeatable (123) as x",
+      "output": {
+        "failure": "invalid argument: found bernoulli at 28:37 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (-10 percent) as x",
+      "exception": "\nSampling fraction (-0.1) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (-10 percent) as x\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (0 percent) as x",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (0.1 percent) as x",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (100 percent) as x",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (100 rows)",
+      "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support ROWS sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (100 rows)\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (150 percent) as x",
+      "exception": "\nSampling fraction (1.5) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (150 percent) as x\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (300M)",
+      "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BYTES sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (300M)\n                ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (43 percent) as x",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (43 percent) repeatable (123)",
+      "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_REPEATABLE] The feature is not supported: TABLESAMPLE SYSTEM does not support the REPEATABLE clause. Use TABLESAMPLE BERNOULLI for repeatable sampling with a seed. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (43 percent) repeatable (123)\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (bucket 3 out of 32 on rand())",
+      "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BUCKET sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (bucket 3 out of 32 on rand())\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (bucket 4 out of 10 on x)",
+      "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BUCKET sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (bucket 4 out of 10 on x)\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
+    {
+      "input": "select * from t tablesample system (bucket 4 out of 10)",
+      "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BUCKET sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (bucket 4 out of 10)\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found system at 28:34 expected '('"
+      }
+    },
     {
       "input": "select * from t tablesample(100 rows)",
       "output": {
@@ -4303,8 +5153,132 @@
       }
     },
     {
-      "input": "select * from t tablesample(bucket 11 out of 10) as x",
-      "exception": "\nSampling fraction (1.1) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 11 out of 10) as x\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "input": "select * from t tablesample(43 percent) repeatable (10) as x",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "t"
+                        ],
+                        "temporal": null,
+                        "sample": {
+                          "method": {
+                            "percent": {
+                              "value": {
+                                "literal": {
+                                  "int32": {
+                                    "value": 43
+                                  }
+                                }
+                              }
+                            }
+                          },
+                          "seed": 10
+                        },
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "name": "x",
+                  "columns": []
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "select * from t tablesample(bucket 11 out of 10) as x",
+      "exception": "\nSampling fraction (1.1) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 11 out of 10) as x\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "tableAlias": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "t"
+                        ],
+                        "temporal": null,
+                        "sample": {
+                          "method": {
+                            "bucket": {
+                              "numerator": 11,
+                              "denominator": 10
+                            }
+                          },
+                          "seed": null
+                        },
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "name": "x",
+                  "columns": []
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedStar": {
+                    "target": null,
+                    "planId": null,
+                    "wildcardOptions": {
+                      "ilikePattern": null,
+                      "excludeColumns": null,
+                      "exceptColumns": null,
+                      "replaceColumns": null,
+                      "renameColumns": null
+                    }
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "select * from t tablesample(bucket 4 out of 10 on x) as x",
+      "exception": "\nTABLESAMPLE(BUCKET x OUT OF y ON colname) is not supported.\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 4 out of 10 on x) as x\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "failure": "invalid argument: found on at 47:49 expected ')'"
+      }
+    },
+    {
+      "input": "select * from t tablesample(bucket 4 out of 10) as x",
       "output": {
         "success": {
           "query": {
@@ -4321,7 +5295,7 @@
                         "sample": {
                           "method": {
                             "bucket": {
-                              "numerator": 11,
+                              "numerator": 4,
                               "denominator": 10
                             }
                           },
@@ -4360,14 +5334,7 @@
       }
     },
     {
-      "input": "select * from t tablesample(bucket 4 out of 10 on x) as x",
-      "exception": "\nTABLESAMPLE(BUCKET x OUT OF y ON colname) is not supported.\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 4 out of 10 on x) as x\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-      "output": {
-        "failure": "invalid argument: found on at 47:49 expected ')'"
-      }
-    },
-    {
-      "input": "select * from t tablesample(bucket 4 out of 10) as x",
+      "input": "select * from t tablesample(bucket 4 out of 10) repeatable (99) as x",
       "output": {
         "success": {
           "query": {
@@ -4388,7 +5355,7 @@
                               "denominator": 10
                             }
                           },
-                          "seed": null
+                          "seed": 99
                         },
                         "options": []
                       },
@@ -5347,6 +6314,153 @@
         }
       }
     },
+    {
+      "input": "select approx from t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "approx"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "select approx, distance, exact, nearest, similarity from t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "approx"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                },
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "distance"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                },
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "exact"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                },
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "nearest"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                },
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "similarity"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
+    {
+      "input": "select distance from t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "distance"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "select distinct a, b from db.c",
       "output": {
@@ -5402,6 +6516,43 @@
         }
       }
     },
+    {
+      "input": "select exact from t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "exact"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "select from tbl",
       "output": {
@@ -5520,12 +6671,86 @@
         "failure": "invalid argument: found 'S' at 18:21 expected '->', '.', '(', '[', '::', 'ESCAPE', 'IS', 'NOT', 'IN', '*', '/', '%', 'DIV', '+', '-', '||', '>>>', '>>', '<<', '&', '^', '|', '!=', '!>', '!<', '==', '=', '>=', '>', '<=>', '<=', '<>', '<', 'BETWEEN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'SIMILAR', 'AND', 'OR', '=>', ',', 'RESPECT', 'IGNORE', or ')'"
       }
     },
+    {
+      "input": "select nearest from t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "nearest"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "select rtrim(trailing 'S' from 'SS abc S'",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'from'. SQLSTATE: 42601 (line 1, pos 26)\n\n== SQL ==\nselect rtrim(trailing 'S' from 'SS abc S'\n--------------------------^^^\n",
       "output": {
         "failure": "invalid argument: found 'S' at 22:25 expected '->', '.', '(', '[', '::', 'ESCAPE', 'IS', 'NOT', 'IN', '*', '/', '%', 'DIV', '+', '-', '||', '>>>', '>>', '<<', '&', '^', '|', '!=', '!>', '!<', '==', '=', '>=', '>', '<=>', '<=', '<>', '<', 'BETWEEN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'SIMILAR', 'AND', 'OR', '=>', ',', 'RESPECT', 'IGNORE', or ')'"
       }
+    },
+    {
+      "input": "select similarity from t",
+      "output": {
+        "success": {
+          "query": {
+            "project": {
+              "input": {
+                "read": {
+                  "namedTable": {
+                    "name": [
+                      "t"
+                    ],
+                    "temporal": null,
+                    "sample": null,
+                    "options": []
+                  },
+                  "isStreaming": false
+                },
+                "planId": null
+              },
+              "expressions": [
+                {
+                  "unresolvedAttribute": {
+                    "name": [
+                      "similarity"
+                    ],
+                    "planId": null,
+                    "isMetadataColumn": false
+                  }
+                }
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
     }
   ]
 }
diff --git a/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json b/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json
index be7b7ff17..749f7cb72 100644
--- a/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json
+++ b/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json
@@ -376,6 +376,116 @@
         }
       }
     },
+    {
+      "input": "with CTE1 (select 1), cte1 as (select 2) select * from cte1",
+      "exception": "\n[DUPLICATED_CTE_NAMES] CTE definition can't have duplicate names: `cte1`. SQLSTATE: 42602\n== SQL (line 1, position 1) ==\nwith CTE1 (select 1), cte1 as (select 2) select * from cte1\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "output": {
+        "success": {
+          "query": {
+            "withCtes": {
+              "input": {
+                "project": {
+                  "input": {
+                    "read": {
+                      "namedTable": {
+                        "name": [
+                          "cte1"
+                        ],
+                        "temporal": null,
+                        "sample": null,
+                        "options": []
+                      },
+                      "isStreaming": false
+                    },
+                    "planId": null
+                  },
+                  "expressions": [
+                    {
+                      "unresolvedStar": {
+                        "target": null,
+                        "planId": null,
+                        "wildcardOptions": {
+                          "ilikePattern": null,
+                          "excludeColumns": null,
+                          "exceptColumns": null,
+                          "replaceColumns": null,
+                          "renameColumns": null
+                        }
+                      }
+                    }
+                  ]
+                },
+                "planId": null
+              },
+              "recursive": false,
+              "ctes": [
+                [
+                  "CTE1",
+                  {
+                    "tableAlias": {
+                      "input": {
+                        "project": {
+                          "input": {
+                            "empty": {
+                              "produceOneRow": true
+                            },
+                            "planId": null
+                          },
+                          "expressions": [
+                            {
+                              "literal": {
+                                "int32": {
+                                  "value": 1
+                                }
+                              }
+                            }
+                          ]
+                        },
+                        "planId": null
+                      },
+                      "name": "CTE1",
+                      "columns": []
+                    },
+                    "planId": null
+                  }
+                ],
+                [
+                  "cte1",
+                  {
+                    "tableAlias": {
+                      "input": {
+                        "project": {
+                          "input": {
+                            "empty": {
+                              "produceOneRow": true
+                            },
+                            "planId": null
+                          },
+                          "expressions": [
+                            {
+                              "literal": {
+                                "int32": {
+                                  "value": 2
+                                }
+                              }
+                            }
+                          ]
+                        },
+                        "planId": null
+                      },
+                      "name": "cte1",
+                      "columns": []
+                    },
+                    "planId": null
+                  }
+                ]
+              ]
+            },
+            "planId": null
+          }
+        }
+      }
+    },
     {
       "input": "with cte1 (select 1) select * from cte1",
       "output": {
diff --git a/crates/sail-spark-connect/tests/gold_data/table_schema.json b/crates/sail-spark-connect/tests/gold_data/table_schema.json
index 9706c3411..d45d26539 100644
--- a/crates/sail-spark-connect/tests/gold_data/table_schema.json
+++ b/crates/sail-spark-connect/tests/gold_data/table_schema.json
@@ -45,7 +45,7 @@
       "input": "a",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near end of input. SQLSTATE: 42601 (line 1, pos 1)\n\n== SQL ==\na\n-^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found > at 8:9 expected ':', or data type"
       }
     },
     {
@@ -67,28 +67,28 @@
       "input": "a INT b long",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'b'. SQLSTATE: 42601 (line 1, pos 6)\n\n== SQL ==\na INT b long\n------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found b at 13:14 expected 'NOT', 'COMMENT', ',', or '>'"
       }
     },
     {
       "input": "a INT, b long, c int,",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near end of input. SQLSTATE: 42601 (line 1, pos 21)\n\n== SQL ==\na INT, b long, c int,\n---------------------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found > at 28:29 expected identifier"
       }
     },
     {
       "input": "a INT, b long,,",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near ','. SQLSTATE: 42601 (line 1, pos 14)\n\n== SQL ==\na INT, b long,,\n--------------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found , at 21:22 expected identifier"
       }
     },
     {
       "input": "a INT,, b long",
       "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near ','. SQLSTATE: 42601 (line 1, pos 6)\n\n== SQL ==\na INT,, b long\n------^^^\n",
       "output": {
-        "failure": "error in JSON serde: expected value at line 1 column 1"
+        "failure": "parse error: error in SQL parser: found , at 13:14 expected identifier"
       }
     },
     {
diff --git a/crates/sail-sql-analyzer/src/expression.rs b/crates/sail-sql-analyzer/src/expression.rs
index 907905670..409d1ac21 100644
--- a/crates/sail-sql-analyzer/src/expression.rs
+++ b/crates/sail-sql-analyzer/src/expression.rs
@@ -1003,9 +1003,11 @@ fn from_ast_atom_expression(atom: AtomExpr) -> SqlResult {
             let null_treatment = match (inner_null_treatment, null_treatment) {
                 (Some(x), None) => Some(x),
                 (None, Some(x)) => Some(x),
-                (Some(_), Some(_)) => return Err(SqlError::invalid(
-                    "conflicting null treatment clause inside and outside function argument list",
-                )),
+                (Some(_), Some(_)) => {
+                    return Err(SqlError::invalid(
+                        "conflicting null treatment clause inside and outside function argument list",
+                    ));
+                }
                 (None, None) => None,
             };
             let ignore_nulls = match null_treatment {
diff --git a/crates/sail-sql-analyzer/src/literal/datetime.rs b/crates/sail-sql-analyzer/src/literal/datetime.rs
index 8206d72b0..108f31bc8 100644
--- a/crates/sail-sql-analyzer/src/literal/datetime.rs
+++ b/crates/sail-sql-analyzer/src/literal/datetime.rs
@@ -1,9 +1,9 @@
 use std::str::FromStr;
 
 use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::prelude::{any, choice, end, just, one_of};
-use chumsky::Parser;
 
 use crate::error::{SqlError, SqlResult};
 
@@ -35,13 +35,24 @@ impl TryFrom for NaiveTime {
     type Error = SqlError;
 
     fn try_from(value: TimeValue) -> SqlResult {
-        NaiveTime::from_hms_nano_opt(
+        let second = if value.second == 60 {
+            if value.hour != 23 || value.minute != 59 {
+                return Err(SqlError::invalid("leap second must be at 23:59:60"));
+            }
+            59
+        } else {
+            value.second
+        };
+
+        let time = NaiveTime::from_hms_nano_opt(
             value.hour as u32,
             value.minute as u32,
-            value.second as u32,
+            second as u32,
             value.nanoseconds,
         )
-        .ok_or_else(|| SqlError::invalid(format!("{value:?}")))
+        .ok_or_else(|| SqlError::invalid(format!("{value:?}")))?;
+
+        Ok(time)
     }
 }
 
@@ -59,9 +70,18 @@ impl<'a> TimestampValue<'a> {
             time,
             timezone,
         } = self;
+
+        let is_leap_second = time.second == 60;
         let date = NaiveDate::try_from(date)?;
         let time = NaiveTime::try_from(time)?;
         let datetime = date.and_time(time);
+
+        let datetime = if is_leap_second {
+            datetime + chrono::Duration::seconds(1)
+        } else {
+            datetime
+        };
+
         Ok((datetime, timezone))
     }
 }
@@ -100,6 +120,11 @@ where
         .map(|s: &str| s.parse::().unwrap())
 }
 
+/// Parses fractional seconds with configurable digit range.
+///
+/// The parser accepts up to 9 digits (nanoseconds) for compatibility with Spark SQL syntax,
+/// but Spark stores timestamps with microsecond precision (6 digits). Nanosecond part is
+/// truncated during conversion to match Spark behavior.
 #[expect(clippy::unwrap_used)]
 fn fraction<'a, T, E>(min_digits: usize, max_digits: usize) -> impl Parser<'a, &'a str, T, E>
 where
diff --git a/crates/sail-sql-analyzer/src/literal/interval.rs b/crates/sail-sql-analyzer/src/literal/interval.rs
index 99e96bdd0..667761810 100644
--- a/crates/sail-sql-analyzer/src/literal/interval.rs
+++ b/crates/sail-sql-analyzer/src/literal/interval.rs
@@ -11,7 +11,7 @@ use sail_sql_parser::ast::expression::{
 };
 
 use crate::error::{SqlError, SqlResult};
-use crate::literal::utils::{extract_fraction_match, extract_match, parse_signed_value, Signed};
+use crate::literal::utils::{Signed, extract_fraction_match, extract_match, parse_signed_value};
 use crate::parser::parse_interval_literal;
 use crate::value::from_ast_string;
 
diff --git a/crates/sail-sql-analyzer/src/literal/numeric.rs b/crates/sail-sql-analyzer/src/literal/numeric.rs
index 24a70c0c8..8577baa26 100644
--- a/crates/sail-sql-analyzer/src/literal/numeric.rs
+++ b/crates/sail-sql-analyzer/src/literal/numeric.rs
@@ -2,8 +2,8 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use sail_common::spec;
 use sail_common::spec::{
-    i256, ARROW_DECIMAL128_MAX_PRECISION, ARROW_DECIMAL256_MAX_PRECISION,
-    ARROW_DECIMAL256_MAX_SCALE,
+    ARROW_DECIMAL128_MAX_PRECISION, ARROW_DECIMAL256_MAX_PRECISION, ARROW_DECIMAL256_MAX_SCALE,
+    i256,
 };
 
 use crate::error::{SqlError, SqlResult};
@@ -303,14 +303,14 @@ mod tests {
             }
         );
         assert!(parse("123456789012345678901234567890123456789").is_ok());
-        assert!(parse(
-            "1234567890123456789012345678901234567891234567890123456789012345678901234567"
-        )
-        .is_ok());
-        assert!(parse(
-            "12345678901234567890123456789012345678912345678901234567890123456789012345677"
-        )
-        .is_err());
+        assert!(
+            parse("1234567890123456789012345678901234567891234567890123456789012345678901234567")
+                .is_ok()
+        );
+        assert!(
+            parse("12345678901234567890123456789012345678912345678901234567890123456789012345677")
+                .is_err()
+        );
         Ok(())
     }
 }
diff --git a/crates/sail-sql-analyzer/src/parser.rs b/crates/sail-sql-analyzer/src/parser.rs
index 02aa20763..2729fef40 100644
--- a/crates/sail-sql-analyzer/src/parser.rs
+++ b/crates/sail-sql-analyzer/src/parser.rs
@@ -1,6 +1,6 @@
+use chumsky::Parser;
 use chumsky::input::Input;
 use chumsky::span::SimpleSpan;
-use chumsky::Parser;
 use sail_sql_parser::ast::data_type::DataType;
 use sail_sql_parser::ast::expression::{Expr, IntervalLiteral};
 use sail_sql_parser::ast::identifier::{ObjectName, QualifiedWildcard};
@@ -13,14 +13,14 @@ use sail_sql_parser::parser::{
     create_named_expression_parser, create_object_name_parser, create_parser,
     create_qualified_wildcard_parser,
 };
-use sail_sql_parser::token::Token;
+use sail_sql_parser::token::{Punctuation, Token};
 
 use crate::error::{SqlError, SqlResult};
 use crate::literal::datetime::{
-    create_date_parser, create_time_parser, create_timestamp_parser, DateValue, TimeValue,
-    TimestampValue,
+    DateValue, TimeValue, TimestampValue, create_date_parser, create_time_parser,
+    create_timestamp_parser,
 };
-use crate::literal::interval::{parse_unqualified_interval_string, IntervalValue};
+use crate::literal::interval::{IntervalValue, parse_unqualified_interval_string};
 
 fn map_parser_input<'a, C>(
     (t, s): &'a (Token<'a>, SimpleSpan),
@@ -52,6 +52,27 @@ macro_rules! parse_simple {
     }};
 }
 
+pub fn rewrite_positional_parameter_markers(s: &str) -> SqlResult<(String, usize)> {
+    let options = ParserOptions::default();
+    let lexer = create_lexer::<_, chumsky::extra::Err>>(&options);
+    let tokens = lexer.parse(s).into_result().map_err(SqlError::parser)?;
+
+    let mut output = String::with_capacity(s.len());
+    let mut last = 0;
+    let mut count = 0;
+    for (token, span) in tokens {
+        if matches!(token, Token::Punctuation(Punctuation::QuestionMark)) {
+            count += 1;
+            output.push_str(&s[last..span.start]);
+            output.push('$');
+            output.push_str(&count.to_string());
+            last = span.end;
+        }
+    }
+    output.push_str(&s[last..]);
+    Ok((output, count))
+}
+
 pub fn parse_data_type(s: &str) -> SqlResult {
     parse!(s, create_data_type_parser)
 }
diff --git a/crates/sail-sql-analyzer/src/query.rs b/crates/sail-sql-analyzer/src/query.rs
index 84281c84c..f65e0d6a0 100644
--- a/crates/sail-sql-analyzer/src/query.rs
+++ b/crates/sail-sql-analyzer/src/query.rs
@@ -686,7 +686,7 @@ fn query_plan_with_table_modifier(
                     let alias = match alias {
                         Some((_, Either::Left(x))) => Some(x.value.into()),
                         Some((_, Either::Right(IdentList { .. }))) => {
-                            return Err(SqlError::invalid("multiple alias for pivot value"))
+                            return Err(SqlError::invalid("multiple alias for pivot value"));
                         }
                         None => None,
                     };
@@ -695,7 +695,7 @@ fn query_plan_with_table_modifier(
                 .collect::>>()?;
             Ok(spec::QueryPlan::new(spec::QueryNode::Pivot(spec::Pivot {
                 input: Box::new(plan),
-                grouping: vec![],
+                grouping: None,
                 aggregate,
                 columns,
                 values,
diff --git a/crates/sail-sql-analyzer/src/statement.rs b/crates/sail-sql-analyzer/src/statement.rs
index 1e9f1e267..fec7256af 100644
--- a/crates/sail-sql-analyzer/src/statement.rs
+++ b/crates/sail-sql-analyzer/src/statement.rs
@@ -12,12 +12,13 @@ use sail_sql_parser::ast::statement::{
     AsQueryClause, Assignment, AssignmentList, ColumnAlteration, ColumnAlterationList,
     ColumnAlterationOption, ColumnDefinition, ColumnDefinitionList, ColumnDefinitionOption,
     ColumnPosition, ColumnTypeDefinition, CommentValue, CreateDatabaseClause, CreateTableClause,
-    CreateViewClause, CreateViewDefinition, DeleteTableAlias, DescribeItem, ExplainFormat,
-    FileFormat, InsertDirectoryDestination, MergeMatchClause, MergeMatchedAction,
+    CreateViewClause, CreateViewDefinition, DeleteTableAlias, DescribeFunctionName, DescribeItem,
+    ExplainFormat, FileFormat, InsertDirectoryDestination, MergeMatchClause, MergeMatchedAction,
     MergeNotMatchedBySourceAction, MergeNotMatchedByTargetAction, MergeSource, PartitionByItem,
     PartitionByList, PartitionClause, PartitionValue, PartitionValueList, PropertyKey,
     PropertyKeyList, PropertyKeyValue, PropertyList, PropertyValue, RowFormat,
-    RowFormatDelimitedClause, SetClause, SortColumn, SortColumnClause, SortColumnList, Statement,
+    RowFormatDelimitedClause, SetClause, ShowFunctionScope, ShowFunctionsClause,
+    ShowFunctionsPattern, SortColumn, SortColumnClause, SortColumnList, Statement,
     TableColumnIdentityOption, TableColumnIdentityOptions, UpdateTableAlias, ViewColumn,
     ViewColumnList, ViewUsingClause,
 };
@@ -32,6 +33,72 @@ use crate::expression::{
 use crate::query::from_ast_query;
 use crate::value::from_ast_string;
 
+fn from_ast_show_function_scope(scope: Option) -> (bool, bool) {
+    match scope {
+        None | Some(ShowFunctionScope::All(_)) => (true, true),
+        Some(ShowFunctionScope::User(_)) => (true, false),
+        Some(ShowFunctionScope::System(_)) => (false, true),
+    }
+}
+
+fn from_ast_show_functions_clause(
+    clause: Option,
+) -> SqlResult<(Option, Option)> {
+    let Some(clause) = clause else {
+        return Ok((None, None));
+    };
+    match clause {
+        ShowFunctionsClause::NamespacePattern(_, database, _, pattern) => Ok((
+            Some(from_ast_object_name(database)?),
+            Some(from_ast_string(pattern)?),
+        )),
+        ShowFunctionsClause::Namespace(_, database) => {
+            Ok((Some(from_ast_object_name(database)?), None))
+        }
+        ShowFunctionsClause::Pattern(_, ShowFunctionsPattern::String(pattern)) => {
+            Ok((None, Some(from_ast_string(pattern)?)))
+        }
+        ShowFunctionsClause::Pattern(_, ShowFunctionsPattern::Name(pattern)) => {
+            let mut parts: Vec = from_ast_object_name(pattern)?.into();
+            let Some(pattern) = parts.pop() else {
+                return Err(SqlError::invalid("SHOW FUNCTIONS with empty pattern"));
+            };
+            Ok((None, Some(pattern)))
+        }
+    }
+}
+
+fn from_ast_describe_function_name(name: DescribeFunctionName) -> SqlResult {
+    let name = match name {
+        DescribeFunctionName::Name(name) => return from_ast_object_name(name),
+        DescribeFunctionName::String(name) => from_ast_string(name)?,
+        DescribeFunctionName::TripleGreaterThan(name) => name.text().trim().to_string(),
+        DescribeFunctionName::DoubleVerticalBar(name) => name.text().trim().to_string(),
+        DescribeFunctionName::DoubleGreaterThan(name) => name.text().trim().to_string(),
+        DescribeFunctionName::DoubleLessThan(name) => name.text().trim().to_string(),
+        DescribeFunctionName::GreaterThanEquals(name) => name.text().trim().to_string(),
+        DescribeFunctionName::LessThanEquals(name) => name.text().trim().to_string(),
+        DescribeFunctionName::LessThanGreaterThan(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Spaceship(name) => name.text().trim().to_string(),
+        DescribeFunctionName::NotEquals(name) => name.text().trim().to_string(),
+        DescribeFunctionName::DoubleEquals(name) => name.text().trim().to_string(),
+        DescribeFunctionName::ExclamationMark(name) => name.text().trim().to_string(),
+        DescribeFunctionName::GreaterThan(name) => name.text().trim().to_string(),
+        DescribeFunctionName::LessThan(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Plus(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Minus(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Asterisk(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Slash(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Percent(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Ampersand(name) => name.text().trim().to_string(),
+        DescribeFunctionName::VerticalBar(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Caret(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Tilde(name) => name.text().trim().to_string(),
+        DescribeFunctionName::Equals(name) => name.text().trim().to_string(),
+    };
+    Ok(spec::ObjectName::bare(name))
+}
+
 /// Converts a parsed SQL AST statement into a spec plan (either a query or a command).
 pub fn from_ast_statement(statement: Statement) -> SqlResult {
     match statement {
@@ -109,7 +176,7 @@ pub fn from_ast_statement(statement: Statement) -> SqlResult {
         } => {
             let cascade = match specifier {
                 Some(Either::Left(Restrict { .. })) => {
-                    return Err(SqlError::todo("RESTRICT in DROP DATABASE"))
+                    return Err(SqlError::todo("RESTRICT in DROP DATABASE"));
                 }
                 Some(Either::Right(Cascade { .. })) => true,
                 None => false,
@@ -533,7 +600,22 @@ pub fn from_ast_statement(statement: Statement) -> SqlResult {
             };
             Ok(spec::Plan::Command(spec::CommandPlan::new(node)))
         }
-        Statement::ShowFunctions { .. } => Err(SqlError::todo("SHOW FUNCTIONS")),
+        Statement::ShowFunctions {
+            show: _,
+            scope,
+            functions: _,
+            clause,
+        } => {
+            let (database, pattern) = from_ast_show_functions_clause(clause)?;
+            let (show_user_functions, show_system_functions) = from_ast_show_function_scope(scope);
+            let node = spec::CommandNode::ShowFunctions {
+                database,
+                pattern,
+                show_user_functions,
+                show_system_functions,
+            };
+            Ok(spec::Plan::Command(spec::CommandPlan::new(node)))
+        }
         Statement::Explain {
             explain: _,
             format,
@@ -1096,18 +1178,10 @@ pub fn from_ast_statement(statement: Statement) -> SqlResult {
                     function: _,
                     extended,
                     item,
-                } => {
-                    let function = match item {
-                        Either::Left(x @ ObjectName { .. }) => from_ast_object_name(x)?,
-                        Either::Right(x @ StringLiteral { .. }) => {
-                            spec::ObjectName::bare(from_ast_string(x)?)
-                        }
-                    };
-                    spec::CommandNode::DescribeFunction {
-                        function,
-                        extended: extended.is_some(),
-                    }
-                }
+                } => spec::CommandNode::DescribeFunction {
+                    function: from_ast_describe_function_name(item)?,
+                    extended: extended.is_some(),
+                },
                 DescribeItem::Catalog {
                     catalog: _,
                     extended,
@@ -1271,7 +1345,7 @@ fn from_ast_table_definition(
         (None, Some(stored_as)) => Some(from_ast_file_format(stored_as)?),
         (None, None) => None,
         (Some(_), Some(_)) => {
-            return Err(SqlError::invalid("conflicting USING and STORED AS clauses"))
+            return Err(SqlError::invalid("conflicting USING and STORED AS clauses"));
         }
     };
     let partition_by = partition_by
diff --git a/crates/sail-sql-analyzer/src/value.rs b/crates/sail-sql-analyzer/src/value.rs
index 3582db9d2..dc8a925a1 100644
--- a/crates/sail-sql-analyzer/src/value.rs
+++ b/crates/sail-sql-analyzer/src/value.rs
@@ -7,7 +7,7 @@ use crate::error::{SqlError, SqlResult};
 use crate::literal::binary::BinaryValue;
 use crate::literal::numeric::{
     parse_decimal_128_string, parse_decimal_256_string, parse_f32_string, parse_f64_string,
-    parse_i16_string, parse_i32_string, parse_i64_string, parse_i8_string,
+    parse_i8_string, parse_i16_string, parse_i32_string, parse_i64_string,
 };
 
 pub(crate) fn from_ast_number_literal(value: NumberLiteral) -> SqlResult {
diff --git a/crates/sail-sql-macro/src/lib.rs b/crates/sail-sql-macro/src/lib.rs
index 90d6b8210..ac1b55c32 100644
--- a/crates/sail-sql-macro/src/lib.rs
+++ b/crates/sail-sql-macro/src/lib.rs
@@ -2,7 +2,7 @@ extern crate proc_macro;
 extern crate proc_macro2;
 
 use proc_macro::TokenStream;
-use syn::{parse_macro_input, DeriveInput};
+use syn::{DeriveInput, parse_macro_input};
 
 mod attribute;
 mod tree;
diff --git a/crates/sail-sql-macro/src/tree/parser.rs b/crates/sail-sql-macro/src/tree/parser.rs
index f1e34673d..6cdfad401 100644
--- a/crates/sail-sql-macro/src/tree/parser.rs
+++ b/crates/sail-sql-macro/src/tree/parser.rs
@@ -203,7 +203,7 @@ pub(crate) fn derive_tree_parser(input: DeriveInput) -> syn::Result
             return Err(syn::Error::new(
                 input.span(),
                 format!("`{TRAIT}` can only be derived for enums or structs"),
-            ))
+            ));
         }
     };
 
diff --git a/crates/sail-sql-macro/src/tree/syntax.rs b/crates/sail-sql-macro/src/tree/syntax.rs
index 7f90887e0..18976ebc5 100644
--- a/crates/sail-sql-macro/src/tree/syntax.rs
+++ b/crates/sail-sql-macro/src/tree/syntax.rs
@@ -31,7 +31,7 @@ fn derive_fields(spanned: impl Spanned, fields: &Fields) -> syn::Result = fields
diff --git a/crates/sail-sql-macro/src/tree/text.rs b/crates/sail-sql-macro/src/tree/text.rs
index 0b4231870..fea0cbba3 100644
--- a/crates/sail-sql-macro/src/tree/text.rs
+++ b/crates/sail-sql-macro/src/tree/text.rs
@@ -42,7 +42,7 @@ fn derive_fields(spanned: impl Spanned, fields: &Fields) -> syn::Result {
+    ([$(($_:expr_2021, $name:ident),)* $(,)?]) => {
         $(
             #[derive(Debug, Clone)]
             pub struct $name {
diff --git a/crates/sail-sql-parser/src/ast/literal.rs b/crates/sail-sql-parser/src/ast/literal.rs
index 0c7784a41..0890ef22d 100644
--- a/crates/sail-sql-parser/src/ast/literal.rs
+++ b/crates/sail-sql-parser/src/ast/literal.rs
@@ -1,9 +1,9 @@
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::input::{Checkpoint, InputRef, ValueInput};
 use chumsky::inspector::Inspector;
 use chumsky::label::LabelError;
-use chumsky::prelude::{custom, Input};
-use chumsky::Parser;
+use chumsky::prelude::{Input, custom};
 
 use crate::ast::identifier::is_identifier_string;
 use crate::options::ParserOptions;
diff --git a/crates/sail-sql-parser/src/ast/operator.rs b/crates/sail-sql-parser/src/ast/operator.rs
index f5f0f0ef8..d898732bd 100644
--- a/crates/sail-sql-parser/src/ast/operator.rs
+++ b/crates/sail-sql-parser/src/ast/operator.rs
@@ -1,8 +1,8 @@
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::input::{Input, InputRef, ValueInput};
 use chumsky::label::LabelError;
 use chumsky::prelude::custom;
-use chumsky::Parser;
 
 use crate::options::ParserOptions;
 use crate::span::TokenSpan;
diff --git a/crates/sail-sql-parser/src/ast/query.rs b/crates/sail-sql-parser/src/ast/query.rs
index 770002f97..a20e26d5a 100644
--- a/crates/sail-sql-parser/src/ast/query.rs
+++ b/crates/sail-sql-parser/src/ast/query.rs
@@ -1,16 +1,16 @@
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::input::{Input, ValueInput};
 use chumsky::label::LabelError;
 use chumsky::pratt::{infix, left};
 use chumsky::prelude::choice;
-use chumsky::Parser;
 use either::Either;
 use sail_sql_macro::{TreeParser, TreeSyntax, TreeText};
 
 use crate::ast::expression::{
     DuplicateTreatment, Expr, FunctionArgument, GroupingExpr, OrderByExpr, WindowSpec,
 };
-use crate::ast::identifier::{column_ident, object_name, table_ident, Ident, ObjectName};
+use crate::ast::identifier::{Ident, ObjectName, column_ident, object_name, table_ident};
 use crate::ast::keywords::{
     All, Anti, As, Bucket, By, Cluster, Cross, Cube, Distinct, Distribute, Except, Exclude, For,
     From, Full, Group, Having, Identifier, In, Include, Inner, Intersect, Join, Lateral, Left,
diff --git a/crates/sail-sql-parser/src/ast/statement.rs b/crates/sail-sql-parser/src/ast/statement.rs
index ac1e549fc..5f70e6b0c 100644
--- a/crates/sail-sql-parser/src/ast/statement.rs
+++ b/crates/sail-sql-parser/src/ast/statement.rs
@@ -4,7 +4,7 @@ use sail_sql_macro::{TreeParser, TreeSyntax, TreeText};
 use crate::ast;
 use crate::ast::data_type::DataType;
 use crate::ast::expression::{BooleanLiteral, Expr, OrderDirection};
-use crate::ast::identifier::{table_ident, Ident, ObjectName};
+use crate::ast::identifier::{Ident, ObjectName, table_ident};
 use crate::ast::keywords::{
     Add, After, All, Alter, Always, Analyze, And, As, Buckets, By, Cache, Cascade, Catalog,
     Catalogs, Change, Check, Clear, Cluster, Clustered, Codegen, Collection, Column, Columns,
@@ -16,13 +16,16 @@ use crate::ast::keywords::{
     Name, Namespace, Namespaces, Noscan, Not, Null, On, Options, Or, Outputformat, Overwrite,
     Partition, Partitioned, Partitions, Properties, Purge, Recover, Refresh, Rename, Replace,
     Restrict, Row, Schema, Schemas, Serde, Serdeproperties, Set, Show, Sorted, Source, Start,
-    Statistics, Stored, Table, Tables, Target, Tblproperties, Temp, Temporary, Terminated, Then,
-    Time, To, Type, Uncache, Unset, Update, Use, Using, Values, Verbose, View, Views, When, With,
-    Zone,
+    Statistics, Stored, System, Table, Tables, Target, Tblproperties, Temp, Temporary, Terminated,
+    Then, Time, To, Type, Uncache, Unset, Update, Use, User, Using, Values, Verbose, View, Views,
+    When, With, Zone,
 };
 use crate::ast::literal::{IntegerLiteral, NumberLiteral, StringLiteral};
 use crate::ast::operator::{
-    Asterisk, Colon, Comma, Equals, ExclamationMark, LeftParenthesis, Minus, Plus, RightParenthesis,
+    Ampersand, Asterisk, Caret, Colon, Comma, DoubleEquals, DoubleGreaterThan, DoubleLessThan,
+    DoubleVerticalBar, Equals, ExclamationMark, GreaterThan, GreaterThanEquals, LeftParenthesis,
+    LessThan, LessThanEquals, LessThanGreaterThan, Minus, NotEquals, Percent, Plus,
+    RightParenthesis, Slash, Spaceship, Tilde, TripleGreaterThan, VerticalBar,
 };
 use crate::ast::query::{AliasClause, IdentList, Query, WhereClause};
 use crate::combinator::{boxed, compose, sequence, unit};
@@ -193,7 +196,9 @@ pub enum Statement {
     },
     ShowFunctions {
         show: Show,
+        scope: Option,
         functions: Functions,
+        clause: Option,
     },
     Explain {
         explain: Explain,
@@ -571,6 +576,26 @@ pub enum DatabaseKeyword {
     Namespace(Namespace),
 }
 
+#[derive(Debug, Clone, TreeParser, TreeSyntax, TreeText)]
+pub enum ShowFunctionScope {
+    All(All),
+    User(User),
+    System(System),
+}
+
+#[derive(Debug, Clone, TreeParser, TreeSyntax, TreeText)]
+pub enum ShowFunctionsClause {
+    NamespacePattern(Either, ObjectName, Like, StringLiteral),
+    Namespace(Either, ObjectName),
+    Pattern(Option, ShowFunctionsPattern),
+}
+
+#[derive(Debug, Clone, TreeParser, TreeSyntax, TreeText)]
+pub enum ShowFunctionsPattern {
+    String(StringLiteral),
+    Name(ObjectName),
+}
+
 #[derive(Debug, Clone, TreeParser, TreeSyntax, TreeText)]
 pub enum DatabasesKeyword {
     Databases(Databases),
@@ -1055,7 +1080,7 @@ pub enum DescribeItem {
     Function {
         function: Function,
         extended: Option,
-        item: Either,
+        item: DescribeFunctionName,
     },
     Catalog {
         catalog: Catalog,
@@ -1092,6 +1117,35 @@ pub enum DescribeItem {
     },
 }
 
+#[derive(Debug, Clone, TreeParser, TreeSyntax, TreeText)]
+pub enum DescribeFunctionName {
+    Name(ObjectName),
+    String(StringLiteral),
+    TripleGreaterThan(TripleGreaterThan),
+    DoubleVerticalBar(DoubleVerticalBar),
+    DoubleGreaterThan(DoubleGreaterThan),
+    DoubleLessThan(DoubleLessThan),
+    GreaterThanEquals(GreaterThanEquals),
+    LessThanEquals(LessThanEquals),
+    LessThanGreaterThan(LessThanGreaterThan),
+    Spaceship(Spaceship),
+    NotEquals(NotEquals),
+    DoubleEquals(DoubleEquals),
+    ExclamationMark(ExclamationMark),
+    GreaterThan(GreaterThan),
+    LessThan(LessThan),
+    Plus(Plus),
+    Minus(Minus),
+    Asterisk(Asterisk),
+    Slash(Slash),
+    Percent(Percent),
+    Ampersand(Ampersand),
+    VerticalBar(VerticalBar),
+    Caret(Caret),
+    Tilde(Tilde),
+    Equals(Equals),
+}
+
 #[derive(Debug, Clone, TreeParser, TreeSyntax, TreeText)]
 pub enum CommentValue {
     NotNull(StringLiteral),
diff --git a/crates/sail-sql-parser/src/container/either.rs b/crates/sail-sql-parser/src/container/either.rs
index 03ac5dfc7..185ea55d3 100644
--- a/crates/sail-sql-parser/src/container/either.rs
+++ b/crates/sail-sql-parser/src/container/either.rs
@@ -1,8 +1,8 @@
 use std::any::TypeId;
 
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::prelude::Input;
-use chumsky::Parser;
 use either::Either;
 
 use crate::combinator::either_or;
diff --git a/crates/sail-sql-parser/src/container/option.rs b/crates/sail-sql-parser/src/container/option.rs
index b1848ed11..52816e486 100644
--- a/crates/sail-sql-parser/src/container/option.rs
+++ b/crates/sail-sql-parser/src/container/option.rs
@@ -1,8 +1,8 @@
 use std::any::TypeId;
 
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::prelude::Input;
-use chumsky::Parser;
 
 use crate::options::ParserOptions;
 use crate::tree::{SyntaxDescriptor, SyntaxNode, TreeParser, TreeSyntax, TreeText};
diff --git a/crates/sail-sql-parser/src/container/sequence.rs b/crates/sail-sql-parser/src/container/sequence.rs
index 3896b1bbd..8481d2046 100644
--- a/crates/sail-sql-parser/src/container/sequence.rs
+++ b/crates/sail-sql-parser/src/container/sequence.rs
@@ -1,8 +1,8 @@
 use std::any::TypeId;
 
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::prelude::Input;
-use chumsky::Parser;
 
 use crate::combinator::sequence;
 use crate::common::Sequence;
diff --git a/crates/sail-sql-parser/src/container/tuple.rs b/crates/sail-sql-parser/src/container/tuple.rs
index 4929cf14c..5c7ee3909 100644
--- a/crates/sail-sql-parser/src/container/tuple.rs
+++ b/crates/sail-sql-parser/src/container/tuple.rs
@@ -1,8 +1,8 @@
 use std::any::TypeId;
 
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::prelude::Input;
-use chumsky::Parser;
 use paste::paste;
 
 use crate::options::ParserOptions;
diff --git a/crates/sail-sql-parser/src/lexer.rs b/crates/sail-sql-parser/src/lexer.rs
index 9c488344f..29181a4e1 100644
--- a/crates/sail-sql-parser/src/lexer.rs
+++ b/crates/sail-sql-parser/src/lexer.rs
@@ -1,7 +1,7 @@
 use chumsky::container::OrderedSeq;
 use chumsky::extra::ParserExtra;
 use chumsky::input::{SliceInput, ValueInput};
-use chumsky::prelude::{any, choice, end, just, none_of, recursive, Input};
+use chumsky::prelude::{Input, any, choice, end, just, none_of, recursive};
 use chumsky::{ConfigParser, IterParser, Parser};
 
 use crate::options::ParserOptions;
@@ -248,7 +248,9 @@ where
     })
 }
 
-fn string<'a, I, E>(options: &ParserOptions) -> impl Parser<'a, I, (Token<'a>, I::Span), E>
+fn string<'a, I, E>(
+    options: &ParserOptions,
+) -> impl Parser<'a, I, (Token<'a>, I::Span), E> + use<'a, I, E>
 where
     I: Input<'a, Token = char> + ValueInput<'a> + SliceInput<'a, Slice = &'a str>,
     E: ParserExtra<'a, I> + 'a,
@@ -270,7 +272,7 @@ where
         dollar_quoted_string(),
     ));
 
-    let string = if options.allow_triple_quote_string {
+    if options.allow_triple_quote_string {
         choice((
             // Multi-quote delimiter must come before one-quote delimiter.
             raw_string("'''", |prefix| StringStyle::TripleSingleQuoted { prefix }),
@@ -288,14 +290,12 @@ where
         .boxed()
     } else {
         string.boxed()
-    };
-
-    string
+    }
 }
 
 pub fn create_lexer<'a, I, E>(
     options: &ParserOptions,
-) -> impl Parser<'a, I, Vec<(Token<'a>, I::Span)>, E>
+) -> impl Parser<'a, I, Vec<(Token<'a>, I::Span)>, E> + use<'a, I, E>
 where
     I: Input<'a, Token = char> + ValueInput<'a> + SliceInput<'a, Slice = &'a str>,
     E: ParserExtra<'a, I> + 'a,
diff --git a/crates/sail-sql-parser/src/parser.rs b/crates/sail-sql-parser/src/parser.rs
index 5c8d7c496..cd992bf31 100644
--- a/crates/sail-sql-parser/src/parser.rs
+++ b/crates/sail-sql-parser/src/parser.rs
@@ -1,7 +1,7 @@
 use chumsky::extra::ParserExtra;
 use chumsky::input::{Input, ValueInput};
 use chumsky::label::LabelError;
-use chumsky::prelude::{end, Recursive};
+use chumsky::prelude::{Recursive, end};
 use chumsky::{IterParser, Parser};
 
 use crate::ast::data_type::DataType;
diff --git a/crates/sail-sql-parser/src/string.rs b/crates/sail-sql-parser/src/string.rs
index 61b6aa6d0..4f3ea6f85 100644
--- a/crates/sail-sql-parser/src/string.rs
+++ b/crates/sail-sql-parser/src/string.rs
@@ -277,10 +277,10 @@ enum Char<'a> {
 
 impl<'a> Char<'a> {
     fn from_str(value: &str, raw: &'a str) -> Char<'a> {
-        if let Ok(n) = u32::from_str_radix(value, RADIX) {
-            if let Some(c) = char::from_u32(n) {
-                return Char::One(c);
-            }
+        if let Ok(n) = u32::from_str_radix(value, RADIX)
+            && let Some(c) = char::from_u32(n)
+        {
+            return Char::One(c);
         }
         Char::Invalid(raw)
     }
diff --git a/crates/sail-sql-parser/src/token.rs b/crates/sail-sql-parser/src/token.rs
index 5a2b4348a..5826fa072 100644
--- a/crates/sail-sql-parser/src/token.rs
+++ b/crates/sail-sql-parser/src/token.rs
@@ -236,7 +236,7 @@ macro_rules! punctuation_enum {
 for_all_punctuations!(punctuation_enum);
 
 macro_rules! keyword_enum {
-    ([$(($string:expr, $identifier:ident),)* $(,)?]) => {
+    ([$(($string:expr_2021, $identifier:ident),)* $(,)?]) => {
         /// A SQL keyword.
         #[derive(Debug, Clone, Copy, PartialEq, Eq)]
         pub enum Keyword {
@@ -471,7 +471,7 @@ impl Keyword {
 #[cfg(test)]
 mod tests {
     macro_rules! keyword_values {
-        ([$(($string:expr, $_:ident),)* $(,)?]) => {
+        ([$(($string:expr_2021, $_:ident),)* $(,)?]) => {
             static KEYWORD_VALUES: &[&str] = &[ $($string,)* ];
         };
     }
diff --git a/crates/sail-sql-parser/src/tree/parser.rs b/crates/sail-sql-parser/src/tree/parser.rs
index 3ed62a2c0..d46454a0a 100644
--- a/crates/sail-sql-parser/src/tree/parser.rs
+++ b/crates/sail-sql-parser/src/tree/parser.rs
@@ -1,6 +1,6 @@
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::input::Input;
-use chumsky::Parser;
 
 use crate::options::ParserOptions;
 
diff --git a/crates/sail-sql-parser/src/utils.rs b/crates/sail-sql-parser/src/utils.rs
index f39df2c63..2714b1445 100644
--- a/crates/sail-sql-parser/src/utils.rs
+++ b/crates/sail-sql-parser/src/utils.rs
@@ -1,7 +1,7 @@
+use chumsky::Parser;
 use chumsky::extra::ParserExtra;
 use chumsky::input::{Input, InputRef, ValueInput};
 use chumsky::prelude::any;
-use chumsky::Parser;
 
 use crate::token::Token;
 
diff --git a/crates/sail-sql-parser/tests/gold_data/syntax.json b/crates/sail-sql-parser/tests/gold_data/syntax.json
index 2990f6e36..7c148a933 100644
--- a/crates/sail-sql-parser/tests/gold_data/syntax.json
+++ b/crates/sail-sql-parser/tests/gold_data/syntax.json
@@ -2452,6 +2452,188 @@
                 ]
               }
             },
+            {
+              "name": "DescribeFunctionName",
+              "node": {
+                "choice": [
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "ObjectName"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "StringLiteral"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(TripleGreaterThan)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(DoubleVerticalBar)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(DoubleGreaterThan)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(DoubleLessThan)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(GreaterThanEquals)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(LessThanEquals)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(LessThanGreaterThan)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Spaceship)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(NotEquals)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(DoubleEquals)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(ExclamationMark)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(GreaterThan)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(LessThan)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Plus)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Minus)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Asterisk)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Slash)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Percent)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Ampersand)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(VerticalBar)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Caret)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Tilde)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Operator(Equals)"
+                      }
+                    ]
+                  }
+                ]
+              }
+            },
             {
               "name": "DescribeItem",
               "node": {
@@ -2465,7 +2647,7 @@
                         "nonTerminal": "Option(Keyword(Extended))"
                       },
                       {
-                        "nonTerminal": "Either(ObjectName, StringLiteral)"
+                        "nonTerminal": "DescribeFunctionName"
                       }
                     ]
                   },
@@ -2798,19 +2980,6 @@
                 ]
               }
             },
-            {
-              "name": "Either(ObjectName, StringLiteral)",
-              "node": {
-                "choice": [
-                  {
-                    "nonTerminal": "ObjectName"
-                  },
-                  {
-                    "nonTerminal": "StringLiteral"
-                  }
-                ]
-              }
-            },
             {
               "name": "Either(Operator(Plus), Operator(Minus))",
               "node": {
@@ -5930,6 +6099,14 @@
                 }
               }
             },
+            {
+              "name": "Keyword(System)",
+              "node": {
+                "terminal": {
+                  "keyword": "SYSTEM"
+                }
+              }
+            },
             {
               "name": "Keyword(SystemTime)",
               "node": {
@@ -6218,6 +6395,14 @@
                 }
               }
             },
+            {
+              "name": "Keyword(User)",
+              "node": {
+                "terminal": {
+                  "keyword": "USER"
+                }
+              }
+            },
             {
               "name": "Keyword(Using)",
               "node": {
@@ -7595,6 +7780,22 @@
                 }
               }
             },
+            {
+              "name": "Option(ShowFunctionScope)",
+              "node": {
+                "optional": {
+                  "nonTerminal": "ShowFunctionScope"
+                }
+              }
+            },
+            {
+              "name": "Option(ShowFunctionsClause)",
+              "node": {
+                "optional": {
+                  "nonTerminal": "ShowFunctionsClause"
+                }
+              }
+            },
             {
               "name": "Option(SortColumnClause)",
               "node": {
@@ -9228,6 +9429,98 @@
                 ]
               }
             },
+            {
+              "name": "ShowFunctionScope",
+              "node": {
+                "choice": [
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Keyword(All)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Keyword(User)"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Keyword(System)"
+                      }
+                    ]
+                  }
+                ]
+              }
+            },
+            {
+              "name": "ShowFunctionsClause",
+              "node": {
+                "choice": [
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Either(Keyword(From), Keyword(In))"
+                      },
+                      {
+                        "nonTerminal": "ObjectName"
+                      },
+                      {
+                        "nonTerminal": "Keyword(Like)"
+                      },
+                      {
+                        "nonTerminal": "StringLiteral"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Either(Keyword(From), Keyword(In))"
+                      },
+                      {
+                        "nonTerminal": "ObjectName"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "Option(Keyword(Like))"
+                      },
+                      {
+                        "nonTerminal": "ShowFunctionsPattern"
+                      }
+                    ]
+                  }
+                ]
+              }
+            },
+            {
+              "name": "ShowFunctionsPattern",
+              "node": {
+                "choice": [
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "StringLiteral"
+                      }
+                    ]
+                  },
+                  {
+                    "sequence": [
+                      {
+                        "nonTerminal": "ObjectName"
+                      }
+                    ]
+                  }
+                ]
+              }
+            },
             {
               "name": "SortByClause",
               "node": {
@@ -9703,8 +9996,14 @@
                       {
                         "nonTerminal": "Keyword(Show)"
                       },
+                      {
+                        "nonTerminal": "Option(ShowFunctionScope)"
+                      },
                       {
                         "nonTerminal": "Keyword(Functions)"
+                      },
+                      {
+                        "nonTerminal": "Option(ShowFunctionsClause)"
                       }
                     ]
                   },
diff --git a/crates/sail-telemetry/Cargo.toml b/crates/sail-telemetry/Cargo.toml
index b4859dd38..a8ede5f89 100644
--- a/crates/sail-telemetry/Cargo.toml
+++ b/crates/sail-telemetry/Cargo.toml
@@ -22,8 +22,6 @@ opentelemetry-appender-log = { workspace = true }
 log = { workspace = true }
 datafusion = { workspace = true }
 futures = { workspace = true }
-tower = { workspace = true }
-tonic = { workspace = true }
 pin-project-lite = { workspace = true }
 num-traits = { workspace = true }
 tokio = { workspace = true }
diff --git a/crates/sail-telemetry/build.rs b/crates/sail-telemetry/build.rs
index 647b2bf78..fd1fa489b 100644
--- a/crates/sail-telemetry/build.rs
+++ b/crates/sail-telemetry/build.rs
@@ -148,7 +148,7 @@ fn build_metric_registry() -> Result<(), Box> {
                     return Err(syn::Error::new(
                         proc_macro2::Span::call_site(),
                         format!("invalid metric: {t:?} {vt:?}"),
-                    ))
+                    ));
                 }
             };
             let debug = if metric.debug {
@@ -246,7 +246,7 @@ fn build_metric_registry() -> Result<(), Box> {
                     return Err(syn::Error::new(
                         proc_macro2::Span::call_site(),
                         format!("invalid metric: {t:?} {vt:?}"),
-                    ))
+                    ));
                 }
             };
             let debug = if metric.debug {
diff --git a/crates/sail-telemetry/src/execution/join_set.rs b/crates/sail-telemetry/src/execution/join_set.rs
index a2b03edfe..f6af82af9 100644
--- a/crates/sail-telemetry/src/execution/join_set.rs
+++ b/crates/sail-telemetry/src/execution/join_set.rs
@@ -1,8 +1,8 @@
 use std::any::Any;
 
 use datafusion::common::runtime::JoinSetTracer;
-use fastrace::future::FutureExt;
 use fastrace::Span;
+use fastrace::future::FutureExt;
 use futures::future::BoxFuture;
 
 /// A DataFusion [`JoinSetTracer`] implementation for Fastrace.
diff --git a/crates/sail-telemetry/src/execution/metrics/default.rs b/crates/sail-telemetry/src/execution/metrics/default.rs
index eec95f473..d9a565c49 100644
--- a/crates/sail-telemetry/src/execution/metrics/default.rs
+++ b/crates/sail-telemetry/src/execution/metrics/default.rs
@@ -1,7 +1,7 @@
-use datafusion::physical_plan::metrics::{Label, MetricValue};
 use datafusion::physical_plan::Metric;
+use datafusion::physical_plan::metrics::{Label, MetricValue};
+use sail_common::telemetry::KeyValue;
 
-use crate::common::KeyValue;
 use crate::execution::metrics::{MetricEmitter, MetricHandled};
 use crate::metrics::{MetricAttribute, MetricRegistry};
 
diff --git a/crates/sail-telemetry/src/execution/metrics/filter.rs b/crates/sail-telemetry/src/execution/metrics/filter.rs
index d6b822cfa..f9378c6d6 100644
--- a/crates/sail-telemetry/src/execution/metrics/filter.rs
+++ b/crates/sail-telemetry/src/execution/metrics/filter.rs
@@ -1,7 +1,7 @@
-use datafusion::physical_plan::metrics::MetricValue;
 use datafusion::physical_plan::Metric;
+use datafusion::physical_plan::metrics::MetricValue;
+use sail_common::telemetry::KeyValue;
 
-use crate::common::KeyValue;
 use crate::execution::metrics::{MetricEmitter, MetricHandled};
 use crate::metrics::{MetricAttribute, MetricRegistry};
 
diff --git a/crates/sail-telemetry/src/execution/metrics/join.rs b/crates/sail-telemetry/src/execution/metrics/join.rs
index 96fbeb9f6..8fffc979b 100644
--- a/crates/sail-telemetry/src/execution/metrics/join.rs
+++ b/crates/sail-telemetry/src/execution/metrics/join.rs
@@ -1,7 +1,7 @@
-use datafusion::physical_plan::metrics::MetricValue;
 use datafusion::physical_plan::Metric;
+use datafusion::physical_plan::metrics::MetricValue;
+use sail_common::telemetry::KeyValue;
 
-use crate::common::KeyValue;
 use crate::execution::metrics::{MetricEmitter, MetricHandled};
 use crate::metrics::{MetricAttribute, MetricRegistry};
 
diff --git a/crates/sail-telemetry/src/execution/metrics/mod.rs b/crates/sail-telemetry/src/execution/metrics/mod.rs
index f5ffb0801..c96dd4760 100644
--- a/crates/sail-telemetry/src/execution/metrics/mod.rs
+++ b/crates/sail-telemetry/src/execution/metrics/mod.rs
@@ -15,8 +15,8 @@ use datafusion::physical_plan::joins::{
 use datafusion::physical_plan::projection::ProjectionExec;
 use datafusion::physical_plan::{ExecutionPlan, Metric};
 use paste::paste;
+use sail_common::telemetry::KeyValue;
 
-use crate::common::KeyValue;
 use crate::metrics::MetricRegistry;
 
 /// A flag indicating whether the metric was handled by a metric emitter.
diff --git a/crates/sail-telemetry/src/execution/metrics/projection.rs b/crates/sail-telemetry/src/execution/metrics/projection.rs
index 90da52ab1..562277881 100644
--- a/crates/sail-telemetry/src/execution/metrics/projection.rs
+++ b/crates/sail-telemetry/src/execution/metrics/projection.rs
@@ -2,8 +2,8 @@ use datafusion::physical_expr_common::metrics::MetricValue;
 use datafusion::physical_plan::Metric;
 use lazy_static::lazy_static;
 use regex::Regex;
+use sail_common::telemetry::KeyValue;
 
-use crate::common::KeyValue;
 use crate::execution::metrics::default::LabelExtractor;
 use crate::execution::metrics::{MetricEmitter, MetricHandled};
 use crate::metrics::{MetricAttribute, MetricRegistry};
@@ -63,9 +63,9 @@ mod tests {
     use datafusion::arrow::datatypes::{DataType, Field, Schema};
     use datafusion::common::Result;
     use datafusion::physical_expr::expressions::Column;
+    use datafusion::physical_plan::PhysicalExpr;
     use datafusion::physical_plan::empty::EmptyExec;
     use datafusion::physical_plan::projection::ProjectionExec;
-    use datafusion::physical_plan::PhysicalExpr;
 
     use crate::execution::metrics::testing::MetricEmitterTester;
 
diff --git a/crates/sail-telemetry/src/execution/metrics/testing.rs b/crates/sail-telemetry/src/execution/metrics/testing.rs
index 39e18d27f..5d93eb477 100644
--- a/crates/sail-telemetry/src/execution/metrics/testing.rs
+++ b/crates/sail-telemetry/src/execution/metrics/testing.rs
@@ -2,16 +2,16 @@ use std::borrow::Cow;
 use std::sync::Arc;
 use std::time::Duration;
 
-use datafusion::common::{plan_err, DataFusionError, Result};
+use datafusion::common::{DataFusionError, Result, plan_err};
 use datafusion::execution::TaskContext;
 use datafusion::physical_plan::ExecutionPlan;
 use futures::TryStreamExt;
 use opentelemetry::metrics::MeterProvider;
 use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider};
 
+use crate::TracingExecOptions;
 use crate::execution::physical_plan::TracingExec;
 use crate::metrics::{MetricManager, MetricRegistry};
-use crate::TracingExecOptions;
 
 fn format_raw_metrics(plan: &dyn ExecutionPlan) -> String {
     let Some(metrics) = plan.metrics() else {
diff --git a/crates/sail-telemetry/src/execution/physical_plan.rs b/crates/sail-telemetry/src/execution/physical_plan.rs
index 6586790d6..2c6d00de1 100644
--- a/crates/sail-telemetry/src/execution/physical_plan.rs
+++ b/crates/sail-telemetry/src/execution/physical_plan.rs
@@ -8,14 +8,14 @@ use std::time::{Duration, Instant};
 use datafusion::arrow::array::RecordBatch;
 use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion::common::{plan_err, Result, Statistics};
+use datafusion::common::{Result, Statistics, plan_err};
 use datafusion::config::ConfigOptions;
 use datafusion::execution::{SendableRecordBatchStream, TaskContext};
 use datafusion::physical_expr::{
     Distribution, OrderingRequirements, PhysicalExpr, PhysicalSortExpr,
 };
 use datafusion::physical_plan::execution_plan::{
-    check_default_invariants, CardinalityEffect, InvariantLevel,
+    CardinalityEffect, InvariantLevel, check_default_invariants,
 };
 use datafusion::physical_plan::filter_pushdown::{
     ChildPushdownResult, FilterDescription, FilterPushdownPhase, FilterPushdownPropagation,
@@ -29,9 +29,9 @@ use fastrace::Span;
 use fastrace_futures::StreamExt;
 use futures::Stream;
 use pin_project_lite::pin_project;
+use sail_common::telemetry::{KeyValue, SpanAttribute};
 use sail_common_datafusion::utils::items::ItemTaker;
 
-use crate::common::{KeyValue, SpanAttribute};
 use crate::execution::metrics::MetricEmitter;
 use crate::metrics::{MetricAttribute, MetricManager, MetricRegistry};
 
diff --git a/crates/sail-telemetry/src/lib.rs b/crates/sail-telemetry/src/lib.rs
index 1cef92dfd..708b27237 100644
--- a/crates/sail-telemetry/src/lib.rs
+++ b/crates/sail-telemetry/src/lib.rs
@@ -1,12 +1,8 @@
-pub mod common;
 pub mod error;
 mod execution;
-pub mod futures;
-pub mod layers;
 mod loggers;
 pub mod metrics;
-pub mod recorder;
 pub mod telemetry;
 
-pub use execution::physical_plan::{trace_execution_plan, TracingExecOptions};
+pub use execution::physical_plan::{TracingExecOptions, trace_execution_plan};
 pub use metrics::MetricManager;
diff --git a/crates/sail-telemetry/src/loggers/span.rs b/crates/sail-telemetry/src/loggers/span.rs
index ac5cc7e86..63c24e351 100644
--- a/crates/sail-telemetry/src/loggers/span.rs
+++ b/crates/sail-telemetry/src/loggers/span.rs
@@ -1,5 +1,5 @@
-use fastrace::local::LocalSpan;
 use fastrace::Event;
+use fastrace::local::LocalSpan;
 use log::{Log, Metadata, Record};
 
 /// A logger that logs the record as fastrace span events.
diff --git a/crates/sail-telemetry/src/metrics/instruments.rs b/crates/sail-telemetry/src/metrics/instruments.rs
index 20e140c10..3578a7e75 100644
--- a/crates/sail-telemetry/src/metrics/instruments.rs
+++ b/crates/sail-telemetry/src/metrics/instruments.rs
@@ -4,8 +4,7 @@ use std::sync::Arc;
 use datafusion::physical_plan::metrics::{Count, Time};
 use num_traits::ToPrimitive;
 use opentelemetry::metrics::{HistogramBuilder, InstrumentBuilder};
-
-use crate::common::KeyValue;
+use sail_common::telemetry::KeyValue;
 
 /// A trait for instruments that can emit measurements with attributes.
 pub trait InstrumentEmittable {
diff --git a/crates/sail-telemetry/src/metrics/mod.rs b/crates/sail-telemetry/src/metrics/mod.rs
index 26ed1e793..101dede6c 100644
--- a/crates/sail-telemetry/src/metrics/mod.rs
+++ b/crates/sail-telemetry/src/metrics/mod.rs
@@ -1,6 +1,6 @@
 mod instruments;
 
-mod gen {
+mod r#gen {
     include!(concat!(env!("OUT_DIR"), "/metric_registry.rs"));
     include!(concat!(env!("OUT_DIR"), "/metric_attributes.rs"));
 }
@@ -9,7 +9,7 @@ use std::fmt;
 use std::sync::Arc;
 use std::time::Duration;
 
-pub use gen::{MetricAttribute, MetricRegistry};
+pub use r#gen::{MetricAttribute, MetricRegistry};
 pub use instruments::*;
 
 /// Encapsulates a [`MetricRegistry`] together with the metrics collection interval.
diff --git a/crates/sail-telemetry/src/telemetry.rs b/crates/sail-telemetry/src/telemetry.rs
index 1cc8b406b..01a59c7fe 100644
--- a/crates/sail-telemetry/src/telemetry.rs
+++ b/crates/sail-telemetry/src/telemetry.rs
@@ -7,14 +7,14 @@ use std::time::Duration;
 use datafusion::common::runtime::set_join_set_tracer;
 use fastrace::collector::{Config, Reporter, SpanRecord};
 use fastrace_opentelemetry::OpenTelemetryReporter;
-use log::{debug, Log};
+use log::{Log, debug};
 use opentelemetry::metrics::Meter;
-use opentelemetry::{global, InstrumentationScope};
+use opentelemetry::{InstrumentationScope, global};
 use opentelemetry_appender_log::OpenTelemetryLogBridge;
 use opentelemetry_otlp::{LogExporter, Protocol, WithExportConfig};
-use opentelemetry_sdk::logs::{BatchConfigBuilder, BatchLogProcessor, SdkLoggerProvider};
-use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
 use opentelemetry_sdk::Resource;
+use opentelemetry_sdk::logs::{BatchConfigBuilder, BatchLogProcessor, SdkLoggerProvider};
+use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider, Temporality};
 use sail_common::config::{OtlpProtocol, TelemetryConfig};
 
 use crate::error::{TelemetryError, TelemetryResult};
@@ -85,12 +85,14 @@ fn init_traces(
     _: &mut TelemetryState,
     resource: &ResourceOptions,
 ) -> TelemetryResult<()> {
-    if config.export_traces {
+    if config.export_traces
+        && let Some(endpoint) = &config.exporter.otlp.endpoint
+    {
         let exporter = opentelemetry_otlp::SpanExporter::builder()
             .with_tonic()
-            .with_endpoint(config.otlp_endpoint.clone())
-            .with_protocol(get_otlp_protocol(&config.otlp_protocol))
-            .with_timeout(Duration::from_secs(config.otlp_timeout_secs))
+            .with_endpoint(endpoint.clone())
+            .with_protocol(get_otlp_protocol(&config.exporter.otlp.protocol))
+            .with_timeout(Duration::from_secs(config.exporter.otlp.timeout_secs))
             .build()?;
         let reporter = OpenTelemetryReporter::new(
             exporter,
@@ -114,12 +116,17 @@ fn init_metrics(
     state: &mut TelemetryState,
     resource: &ResourceOptions,
 ) -> TelemetryResult<()> {
-    if config.export_metrics {
+    if config.export_metrics
+        && let Some(endpoint) = &config.exporter.otlp.endpoint
+    {
         let exporter = opentelemetry_otlp::MetricExporter::builder()
             .with_tonic()
-            .with_endpoint(config.otlp_endpoint.clone())
-            .with_protocol(get_otlp_protocol(&config.otlp_protocol))
-            .with_timeout(Duration::from_secs(config.otlp_timeout_secs))
+            .with_endpoint(endpoint.clone())
+            .with_protocol(get_otlp_protocol(&config.exporter.otlp.protocol))
+            .with_timeout(Duration::from_secs(config.exporter.otlp.timeout_secs))
+            // Emit only active metrics. Since metrics may have attributes for sessions or jobs,
+            // we do not want to emit metrics after the session or job has ended.
+            .with_temporality(Temporality::Delta)
             .build()?;
         let reader = PeriodicReader::builder(exporter)
             .with_interval(Duration::from_secs(config.metrics_export_interval_secs))
@@ -161,12 +168,14 @@ fn init_logs(
 
     let mut secondary: Vec> = vec![];
 
-    if config.export_logs {
+    if config.export_logs
+        && let Some(endpoint) = &config.exporter.otlp.endpoint
+    {
         let exporter = LogExporter::builder()
             .with_tonic()
-            .with_endpoint(config.otlp_endpoint.clone())
-            .with_protocol(get_otlp_protocol(&config.otlp_protocol))
-            .with_timeout(Duration::from_secs(config.otlp_timeout_secs))
+            .with_endpoint(endpoint.clone())
+            .with_protocol(get_otlp_protocol(&config.exporter.otlp.protocol))
+            .with_timeout(Duration::from_secs(config.exporter.otlp.timeout_secs))
             .build()?;
         let batch_config = BatchConfigBuilder::default()
             .with_scheduled_delay(Duration::from_secs(config.logs_export_interval_secs))
@@ -189,7 +198,7 @@ fn init_logs(
         secondary.push(Box::new(OpenTelemetryLogBridge::new(&provider)));
         state.logger_provider = Some(provider);
     }
-    if config.export_traces {
+    if config.export_traces && config.exporter.otlp.endpoint.is_some() {
         secondary.push(Box::new(SpanEventLogger));
     }
 
@@ -208,16 +217,16 @@ fn init_datafusion_telemetry() -> TelemetryResult<()> {
 pub fn shutdown_telemetry() {
     debug!("Shutting down OpenTelemetry...");
     fastrace::flush();
-    if let Ok(mut status) = TELEMETRY_STATUS.lock() {
-        if let TelemetryStatus::Initialized(ref state) = *status {
-            if let Some(provider) = &state.meter_provider {
-                let _ = provider.shutdown();
-            }
-            if let Some(provider) = &state.logger_provider {
-                let _ = provider.shutdown();
-            }
-            *status = TelemetryStatus::Finalized;
+    if let Ok(mut status) = TELEMETRY_STATUS.lock()
+        && let TelemetryStatus::Initialized(ref state) = *status
+    {
+        if let Some(provider) = &state.meter_provider {
+            let _ = provider.shutdown();
+        }
+        if let Some(provider) = &state.logger_provider {
+            let _ = provider.shutdown();
         }
+        *status = TelemetryStatus::Finalized;
     }
 }
 
diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile
index 3930a1caa..dfa7cbe0c 100644
--- a/docker/dev/Dockerfile
+++ b/docker/dev/Dockerfile
@@ -1,9 +1,10 @@
-ARG RUST_VERSION=1.95.0
+ARG RUST_VERSION=1.96.0
 ARG RUST_PROFILE=dev
 ARG RUSTFLAGS=""
-ARG PYSPARK_VERSION=4.1.1
+ARG PYSPARK_VERSION=4.2.0
+ARG PYTHON_VERSION=3.14
 
-FROM python:3.14-slim AS builder
+FROM python:${PYTHON_VERSION}-slim AS builder
 
 ARG RUST_VERSION
 ARG RUST_PROFILE
@@ -45,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo/registry/ \
     cargo build -p sail-cli --profile ${RUST_PROFILE} --bins && \
     cp /app/target/${RUST_TARGET_SUBDIR}/sail /usr/local/bin
 
-FROM python:3.14-slim
+FROM python:${PYTHON_VERSION}-slim
 
 ARG PYSPARK_VERSION
 
diff --git a/docker/quickstart/Dockerfile b/docker/quickstart/Dockerfile
index 31ebb1ea2..3b4ea92c1 100644
--- a/docker/quickstart/Dockerfile
+++ b/docker/quickstart/Dockerfile
@@ -1,7 +1,9 @@
-FROM python:3.14-slim
+ARG PYTHON_VERSION=3.14
+
+FROM python:${PYTHON_VERSION}-slim
 
 ARG PYSAIL_VERSION
-ARG PYSPARK_VERSION=4.1.1
+ARG PYSPARK_VERSION=4.2.0
 
 RUN python3 -m pip install --no-cache-dir "pysail==${PYSAIL_VERSION}"
 RUN python3 -m pip install --no-cache-dir "pyspark-client==${PYSPARK_VERSION}"
diff --git a/docker/release/Dockerfile b/docker/release/Dockerfile
index b9ca414c4..d3f114521 100644
--- a/docker/release/Dockerfile
+++ b/docker/release/Dockerfile
@@ -1,10 +1,11 @@
 ARG RELEASE_TAG
-ARG RUST_VERSION=1.95.0
+ARG RUST_VERSION=1.96.0
 ARG RUST_PROFILE=release
 ARG RUSTFLAGS=""
-ARG PYSPARK_VERSION=4.1.1
+ARG PYSPARK_VERSION=4.2.0
+ARG PYTHON_VERSION=3.14
 
-FROM python:3.14-slim AS builder
+FROM python:${PYTHON_VERSION}-slim AS builder
 
 ARG RELEASE_TAG
 ARG RUST_VERSION
@@ -45,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo/registry/ \
     cargo build -p sail-cli --profile ${RUST_PROFILE} --bins && \
     cp /app/target/${RUST_TARGET_SUBDIR}/sail /usr/local/bin
 
-FROM python:3.14-slim
+FROM python:${PYTHON_VERSION}-slim
 
 ARG PYSPARK_VERSION
 
diff --git a/docs/development/build/index.md b/docs/development/build/index.md
index a2077c6a1..09fedb5bc 100644
--- a/docs/development/build/index.md
+++ b/docs/development/build/index.md
@@ -1,6 +1,6 @@
 ---
 title: Building the Project
-rank: 2
+rank: 3
 ---
 
 # Building the Project
diff --git a/docs/development/build/python.md b/docs/development/build/python.md
index 1f4ec10e8..356537cb0 100644
--- a/docs/development/build/python.md
+++ b/docs/development/build/python.md
@@ -102,13 +102,13 @@ Alternatively, you can run the tests against a specific Spark version.
 For example, you can use the following command to install the package in a specific `test` environment.
 
 ```bash
-hatch run test.spark-3.5.7:install-pysail
+hatch run test.spark-3.5.9:install-pysail
 ```
 
 Then you can run the tests against the specific Spark version.
 
 ```bash
-hatch run test.spark-3.5.7:pytest --pyargs pysail
+hatch run test.spark-3.5.9:pytest --pyargs pysail
 ```
 
 ### Testing with JVM Spark
@@ -120,15 +120,15 @@ Note that tests written for extended features of Sail will be skipped in this ca
 
 ```bash
 env SPARK_REMOTE="local" \
-  PYSPARK_SUBMIT_ARGS="--packages org.apache.spark:spark-connect_2.12:3.5.7 pyspark-shell" \
-  hatch run test.spark-3.5.7:pytest --pyargs pysail
+  PYSPARK_SUBMIT_ARGS="--packages org.apache.spark:spark-connect_2.12:3.5.9 pyspark-shell" \
+  hatch run test.spark-3.5.9:pytest --pyargs pysail
 ```
 
 The `PYSPARK_SUBMIT_ARGS` environment variable is no longer needed since Spark 4.0.
 
 ```bash
 env SPARK_REMOTE="local" \
-  hatch run test.spark-4.1.1:pytest --pyargs pysail
+  hatch run test.spark-4.2.0:pytest --pyargs pysail
 ```
 
 ::: info
diff --git a/docs/development/coding-agents/index.md b/docs/development/coding-agents/index.md
new file mode 100644
index 000000000..d80e71745
--- /dev/null
+++ b/docs/development/coding-agents/index.md
@@ -0,0 +1,29 @@
+---
+title: Coding Agents
+rank: 2
+---
+
+# Coding Agents
+
+You can add `AGENTS.md` in the project root directory to provide instructions for coding agents.
+This file is ignored by Git so that you can customize it for your needs.
+
+Here is an example that instructs coding agents to read instructions in the `.github` directory when making changes or reviewing code.
+
+::: warning
+The example may change over time. You would need to maintain your local instructions to keep them up to date.
+:::
+
+```markdown
+Read `.github/copilot-instructions.md` for an overview of the project.
+
+Read `.github/instructions/dev.instructions.md` before making changes in this repository and follow it as the default project guidance.
+
+When performing a code review on Rust changes, also read `.github/instructions/rust-review.instructions.md` and follow it for the review.
+
+When performing a code review on Python changes, also read `.github/instructions/python-review.instructions.md` and follow it for the review.
+
+If multiple instruction files apply, use the more specific file for the task in addition to the default project guidance.
+
+You only need to read the instructions once per session. There is no need to read them upon every task.
+```
diff --git a/docs/development/documentation/index.md b/docs/development/documentation/index.md
index 19d3a2d2a..9f424fc58 100644
--- a/docs/development/documentation/index.md
+++ b/docs/development/documentation/index.md
@@ -1,6 +1,6 @@
 ---
 title: Documentation
-rank: 4
+rank: 5
 ---
 
 # Documentation
diff --git a/docs/development/recipes/hms.md b/docs/development/recipes/hms.md
index d0e0a6f02..cc1b82adf 100644
--- a/docs/development/recipes/hms.md
+++ b/docs/development/recipes/hms.md
@@ -1,165 +1,17 @@
 ---
 title: Hive Metastore
+rank: 65
 ---
 
-# Hive Metastore Development Notes
+# Hive Metastore Development
 
-This page documents the current HMS implementation and test strategy in Sail for contributors and maintainers.
+## Generating GSSAPI Bindings
 
-## Maintainer Summary
-
-The current HMS work provides a metadata-focused compatibility layer for Apache Hive Metastore.
-
-What maintainers can rely on today:
-
-- Plain HMS catalog connectivity over Thrift.
-- Kerberos-secured HMS connectivity over Thrift SASL.
-- Kerberos SASL frame wrapping (`auth-int` / `auth-conf`) with configurable minimum QOP (`min_sasl_qop`).
-- HMS endpoint lists with automatic failover on retryable transport/Thrift errors.
-- Basic metadata CRUD for databases and tables through `HmsCatalogProvider`.
-- A self-contained Kerberos integration harness built with `testcontainers`.
-- CI wiring for the ignored catalog-test lane.
-
-What maintainers should not infer from the current test suite:
-
-- Broad production readiness across HMS distributions and versions.
-- Support for Hive ACID or transactional HMS APIs.
-- Support for delegation tokens or TLS.
-
-## What Green Means
-
-The Kerberos HMS ignored integration test target is:
-
-```bash
-cargo test -p sail-catalog-hms --test kerberos_integration_tests -- --ignored
-```
-
-When this target is green, it means Sail successfully:
-
-- boots a local MIT KDC and Kerberized Hive Metastore container
-- obtains client credentials with `kinit`
-- authenticates to HMS over Thrift SASL using Kerberos
-- performs database create/get/drop operations
-- performs table create/get/drop operations
-- fails cleanly when Kerberos credentials are missing
-
-This is evidence that the Kerberos transport and basic HMS metadata path work end to end.
-
-It is not evidence that Sail supports:
-
-- Hive ACID transaction APIs
-- non-Apache HMS distributions
-- production RDBMS-backed HMS deployments
-
-It is only partial evidence for HA behavior:
-
-- the current test suite covers ordered endpoint failover and retryable transport errors
-- it does not yet prove broader operational behavior such as DNS churn under long-lived production traffic, vendor-specific HA frontends, or multi-region deployment patterns
-
-## Local Verification
-
-### Fast checks
-
-```bash
-cargo test -p sail-catalog-hms --lib
-```
-
-This covers HMS provider validation logic, auth config parsing, Kerberos session behavior, and Thrift SASL handshake unit tests.
-
-### Full Kerberos harness
-
-```bash
-cargo test -p sail-catalog-hms --test kerberos_integration_tests -- --ignored
-```
-
-The preferred local environment is the repo devcontainer because it includes:
-
-- Kerberos client tools
-- the GSSAPI runtime used by the dynamic loader
-- Docker access for `testcontainers`
-
-Host-native macOS runs may work, but Linux in the devcontainer or CI is the supported path for the Kerberos harness.
-
-## CI Behavior
-
-The Kerberos HMS harness runs in the ignored catalog-test lane.
-
-On pull requests, that lane is enabled when either:
-
-- the PR has the `run catalog tests` label
-- the head commit message contains `[catalog test]` or `[catalog tests]`
-
-On pushes to `main`, the catalog-test lane runs automatically.
-
-Relevant workflow locations:
-
-- `.github/workflows/build.yml`
-- `.github/workflows/rust-tests.yml`
-
-The Rust test workflow installs Kerberos runtime/client packages before running ignored tests so the host-side `kinit` step and Sail's dynamic GSSAPI loading both work in CI.
-
-## Harness Layout
-
-The HMS Kerberos harness lives in:
-
-- `crates/sail-catalog-hms/tests/common/mod.rs`
-- `crates/sail-catalog-hms/tests/kerberos_integration_tests.rs`
-- `crates/sail-catalog-hms/tests/fixtures/kerberos-kdc/`
-
-The harness currently:
-
-- builds a local MIT KDC image from repo fixtures
-- provisions an HMS service principal and a client principal
-- starts a Kerberized Hive Metastore container
-- writes active `hive-site.xml` and `core-site.xml` into the container's default config locations
-- uses writable `/tmp`-backed Derby paths for the metastore database during tests
-
-## Current Compatibility Contract
-
-The current HMS provider is intentionally conservative.
-
-Supported contract:
-
-- HMS endpoint list via `uris` (`host:port` or `thrift://host:port`, including comma-flattened entries)
-- endpoint-ordered failover with a per-endpoint connect timeout that defaults to `5s` and can be overridden with `connect_timeout_secs`
-- DNS is re-resolved when Sail builds a new connection for an endpoint instead of pinning the startup-resolved address forever
-- Kerberos principal configured with `auth = "kerberos"` and `kerberos_service_principal`
-- minimum Kerberos SASL QOP via `min_sasl_qop` (`auth`, `auth_int`, `auth_conf`)
-- `_HOST` expansion for the principal based on the currently selected endpoint
-- metadata CRUD for databases, tables, and views
-- retried create/drop mutations normalize `AlreadyExists` and `NotFound` responses when the earlier attempt likely succeeded and only the response was lost
-
-Security guarantees in this contract:
-
-- Downgrade fail-fast: if the configured `min_sasl_qop` cannot be met by the server SASL layer advertisement, transport creation fails before the HMS client is usable.
-- Session-wide enforcement: when `auth_int` or `auth_conf` is negotiated, every subsequent Thrift frame on that connection is wrapped on write and unwrapped on read.
-
-Out of scope for the current implementation:
-
-- transactional HMS methods such as `open_txns`, `lock`, `heartbeat`, `allocate_table_write_ids`, or compaction APIs
-- Hortonworks or other distribution-specific compatibility promises
-- automatic keytab management inside Sail
-
-## Regenerating GSSAPI Bindings
-
-The GSSAPI FFI bindings in `crates/sail-catalog-hms/src/security/gssapi_bindings.rs` were generated by `bindgen` from the system GSSAPI headers. To regenerate for a newer GSSAPI version:
+The GSSAPI FFI bindings in `crates/sail-catalog-hms/src/security/gssapi_bindings.rs` were generated by `bindgen` from the system GSSAPI headers.
+Run the following command to regenerate the bindings for a newer GSSAPI version.
 
 ```bash
 bindgen /usr/include/gssapi.h -o crates/sail-catalog-hms/src/security/gssapi_bindings.rs
 ```
 
-The generated bindings are checked in because the build environment may not have `bindgen` installed. The bindings are portable across architectures (x86_64, ARM, macOS, Linux) because the GSSAPI C ABI uses only standard types (`u32` constants, `*mut c_void` pointers, `usize` lengths) that adapt to the host automatically.
-
-## Guidance for Future Changes
-
-If you extend HMS support, keep these boundaries explicit in docs and reviews:
-
-1. Kerberos auth support is not the same as transactional Hive support.
-2. Passing the current Kerberos harness means the auth and metadata path works, not the full HMS surface.
-3. Compatibility with older Hive or vendor-specific HMS deployments should be treated as a versioned test matrix problem, not assumed from Apache Hive success.
-
-The most likely next areas of work are:
-
-- broader distro/version compatibility testing
-- explicit investigation of transactional HMS APIs
-- deciding whether the reusable Kerberos KDC harness should move into a shared test utility crate if another consumer, such as HDFS integration tests, needs it
+The generated bindings are committed because the build environment may not have `bindgen` installed. The bindings are portable across architectures because the GSSAPI C ABI uses only standard types (e.g., `u32` constants, `*mut c_void` pointers, and `usize` lengths) that adapt to the host automatically.
diff --git a/docs/development/recipes/iceberg-openapi-generation.md b/docs/development/recipes/iceberg-openapi-generation.md
deleted file mode 100644
index 9bb65dee9..000000000
--- a/docs/development/recipes/iceberg-openapi-generation.md
+++ /dev/null
@@ -1,79 +0,0 @@
----
-title: Generating Iceberg REST Catalog Client
-rank: 51
----
-
-# Generating Iceberg REST Catalog Client
-
-This guide explains how to regenerate the Rust client code for the Iceberg REST Catalog from the OpenAPI specification.
-
-## Prerequisites
-
-To regenerate the REST client from the OpenAPI specification, you need to install the OpenAPI Generator CLI:
-
-```bash
-brew install openapi-generator
-```
-
-For other installation methods, see the [OpenAPI Generator CLI Installation Guide](https://openapi-generator.tech/docs/installation/).
-
-## Configuration
-
-Generator configuration is defined in `crates/sail-catalog-iceberg/spec/openapi-generator-config.yaml`.
-
-## Generating the REST Client
-
-The REST client is auto-generated from the [Iceberg REST Catalog OpenAPI spec](https://github.com/lakehq/sail/blob/main/crates/sail-catalog-iceberg/spec/iceberg-rest-catalog.yaml).
-
-To regenerate the client code, run the generation script from the repository root:
-
-```bash
-./crates/sail-catalog-iceberg/spec/generate-client.sh
-```
-
-Or from the spec directory:
-
-```bash
-cd crates/sail-catalog-iceberg/spec
-./generate-client.sh
-```
-
-The script will:
-
-1. Generate Rust client code from the OpenAPI spec using `--schema-mappings` to use custom types from `src/types/`
-2. Extract `apis/` and `models/` directories to `src/`
-3. Format the generated code with `cargo fmt`
-
-The generated code will be placed in `src/apis/` and `src/models/`.
-
-## Schema Mappings
-
-The generator uses custom type mappings to avoid problematic generated code:
-
-- `Type`, `StructType`, `ListType`, `MapType`, `StructField` → `crate::types::{Type,StructType,ListType,MapType, NestedFieldRef}`
-
-## Post-Generation Manual Steps
-
-OpenAPI 3.1 support is still in beta when generating Rust clients with the OpenAPI Generator.
-After running the generation script, you must manually fix the following:
-
-1. In `src/apis/catalog_api_api.rs`:
-
-   - Replace `"{}/v1/{prefix}/` with `"{}/v1{prefix}/`
-   - Replace `crate::apis::urlencode(prefix.unwrap())` with `prefix.map(|p| format!("/{}", crate::apis::urlencode(p))).unwrap_or_default()`
-
-2. In `src/apis/o_auth2_api_api.rs`:
-
-   - Replace `models::models::TokenType` with `models::TokenType`
-
-3. In `src/models/schema.rs`:
-
-   - Replace `models::StructField` with `NestedFieldRef`
-
-4. In `src/models/table_update.rs` and `src/models/view_update.rs`:
-   - The upstream spec declares `TableUpdate`/`ViewUpdate` as a bare `anyOf`
-     with no discriminator (unlike `BaseUpdate`, which has one), so the
-     generator emits a degenerate flat struct with every field of every
-     variant marked required. Replace each generated struct with a
-     hand-written `#[serde(tag = "action")]` enum (mirroring `base_update.rs`),
-     with one variant per action.
diff --git a/scripts/spark-gold-data/README.md b/docs/development/recipes/spark-gold-data.md
similarity index 61%
rename from scripts/spark-gold-data/README.md
rename to docs/development/recipes/spark-gold-data.md
index 2a2d6d4c8..3509994f0 100644
--- a/scripts/spark-gold-data/README.md
+++ b/docs/development/recipes/spark-gold-data.md
@@ -1,12 +1,21 @@
-# Gold Data Generation for Spark Tests
+---
+title: Spark Gold Data
+rank: 32
+---
 
-This directory contains scripts to generate gold data for Spark tests.
+# Working with Spark Gold Data
+
+We use gold data to record Sail's behavior for Spark SQL test cases.
+
+The `scripts/spark-gold-data` directory contains scripts to generate gold data for Spark tests. The gold data files are stored in the `crates/sail-spark-connect/tests/gold_data` directory.
+
+## Overview
 
 The idea of gold data is to keep track of the **actual** output for a set of test cases.
 We can then review test output changes when we modify the code.
 This ensures that we do not alter the behavior of the code in unintended ways.
 
-This is similar to `sqllogictest`, which is designed by SQLite, and whose Rust implementation
+This is similar to `sqllogictest`, which was designed by SQLite and whose Rust implementation
 is used by DataFusion and various other projects.
 However, we choose to use JSON format for the gold data (as opposed to the text format proposed by
 `sqllogictest`), so that we can serialize text containing special characters reliably.
@@ -14,22 +23,7 @@ The JSON format also makes it easier to manipulate the data programmatically.
 
 The Spark project leverages the idea of gold data in its tests as well.
 
-## Bootstrapping Gold Data
-
-In the project root directory, run the following command to collect test cases
-from the Spark project.
-This will rewrite the gold data files with only the test input.
-You should then use the command in the next section to save the test output.
-
-```bash
-scripts/spark-gold-data/bootstrap.sh
-```
-
-This is not part of the daily developer workflow.
-This only needs to be done once, when we update the bootstrap scripts, or when we
-upgrade the Spark version.
-
-## Updating Test Output in Gold Data
+## Updating Test Output
 
 As part of the daily developer workflow, you need to run the following command to
 save the test output every time you modify the behavior of the code.
diff --git a/docs/development/recipes/spark-version-update.md b/docs/development/recipes/spark-version-update.md
new file mode 100644
index 000000000..1499a251f
--- /dev/null
+++ b/docs/development/recipes/spark-version-update.md
@@ -0,0 +1,120 @@
+---
+title: Spark Version Update
+rank: 35
+---
+
+# Updating Spark Version
+
+::: info
+This guide is for the project maintainers.
+:::
+
+This page outlines the work required to support a new Spark version in the codebase.
+Each Spark release may introduce features or incompatibilities with our scripts, so updating the codebase may require manual changes.
+
+This page assumes that the Spark source code is available in the `opt/spark` directory, as described in [Spark Setup](../spark-tests/spark-setup).
+
+## Standard Procedures
+
+### Project Settings
+
+Update the PySpark version pins in `pyproject.toml`.
+
+1. Update the `test` optional dependency when updating the latest supported Spark version.
+2. Update the `default` environment when updating the latest supported Spark version.
+3. Update the `test` environment when adding or updating a supported Spark minor version.
+4. Update the `test-spark` environment when changing a Spark version used for parity tests.
+5. Update the `test-ibis` environment only when changing the Spark patch version used for Ibis tests, unless the Ibis version is also being updated.
+
+### Spark Configuration
+
+The `scripts/spark-config/generate.py` script collects Spark configuration into data files in `crates/sail-spark-connect/data/config`.
+The data files are used to generate the related Rust code.
+
+In the project root directory, run a command like the following for each supported Spark minor version to update its configuration data file.
+
+```bash
+env SPARK_LOCAL_IP=127.0.0.1 \
+  hatch run test.spark-3.5.9:scripts/spark-config/generate.py \
+  -o crates/sail-spark-connect/data/config/spark-3.5.json
+```
+
+Update `crates/sail-spark-connect/build.rs` when adding support for a new Spark minor version.
+
+### Documentation
+
+Please search for all mentions of Spark versions in the documentation in the `docs` directory and update them accordingly.
+
+### GitHub Actions Workflows
+
+Please search for all mentions of Spark versions in the GitHub Actions workflows in the `.github` directory and update them accordingly.
+
+### Spark Connect Protocol Buffers
+
+When updating the latest supported Spark version, copy the `.proto` files from Spark to the `crates/sail-spark-connect/proto` directory.
+You may need to adjust the Rust code to work with the generated Spark Connect gRPC service code.
+
+### Parity Tests
+
+Update the parity test patch whenever changing a Spark version in the `test-spark` environment.
+
+We build patched PySpark packages to run parity tests for the supported Spark major versions.
+
+First, update the patch file. Here are example steps for updating from Spark `3.5.8` to `3.5.9`.
+
+1. Create a branch `patch-v3.5.8` from the `v3.5.8` tag.
+2. Run `git -C opt/spark apply ../../scripts/spark-tests/spark-3.5.8.patch` to apply the current patch. Commit the changes.
+3. Create another branch `patch-v3.5.9` from the `v3.5.9` tag.
+4. Merge branch `patch-v3.5.8` into branch `patch-v3.5.9` and resolve any merge conflicts.
+5. Run `git -C opt/spark diff -p v3.5.9 patch-v3.5.9 > scripts/spark-tests/spark-3.5.9.patch` to generate the new patch.
+6. Remove the old patch `scripts/spark-tests/spark-3.5.8.patch`.
+
+The patch must include all PySpark tests in the built package. Make sure the patched `setup.py` includes all test subpackages in the corresponding Spark version.
+
+The following command verifies that the patched PySpark package can be built successfully for the new Spark version. Note that the command may take a long time to run. For example, in GitHub Actions, it takes about 30 minutes.
+
+```bash
+env SPARK_VERSION=3.5.9 scripts/spark-tests/build-pyspark.sh
+```
+
+### Gold Data
+
+Update the gold data patch whenever changing the Spark version used by `scripts/spark-gold-data/bootstrap.sh`.
+
+First, update the patch file. Here are example steps for updating from Spark `3.5.8` to `3.5.9`.
+
+1. Create a branch `gold-patch-v3.5.8` from the `v3.5.8` tag.
+2. Run `git -C opt/spark apply ../../scripts/spark-gold-data/spark-3.5.8.patch` to apply the current patch. Commit the changes.
+3. Create another branch `gold-patch-v3.5.9` from the `v3.5.9` tag.
+4. Merge branch `gold-patch-v3.5.8` into branch `gold-patch-v3.5.9` and resolve any merge conflicts.
+5. Run `git -C opt/spark diff -p v3.5.9 gold-patch-v3.5.9 > scripts/spark-gold-data/spark-3.5.9.patch` to generate the new patch.
+6. Remove the old patch `scripts/spark-gold-data/spark-3.5.8.patch`.
+
+Then update the Spark version mentioned in `scripts/spark-gold-data/bootstrap.sh`.
+
+In the project root directory, run the following command to collect test cases from the Spark project.
+This rewrites the gold data files with the new test input only.
+
+```bash
+scripts/spark-gold-data/bootstrap.sh
+```
+
+The command requires the `JAVA_HOME` environment variable to be set. On macOS, you can specify the Java installation path in the following way.
+
+```bash
+env JAVA_HOME="$(/usr/libexec/java_home)" scripts/spark-gold-data/bootstrap.sh
+```
+
+Then generate the Sail test output using the following command.
+
+```bash
+env SAIL_UPDATE_GOLD_DATA=1 cargo test
+```
+
+## Non-standard Procedures
+
+Review changes to the PySpark UDF and UDTF code in `worker.py` and related Spark files. Identify version-specific logic and update the codebase accordingly.
+
+Review the patch files closely, especially if they are edited directly after being generated.
+Patch files with ad-hoc edits can have incorrect line numbers or file hashes even if the patch applies cleanly.
+It is recommended to generate the patch again using only `git` commands (`git apply` and `git diff`) to ensure the patch is correct.
diff --git a/docs/development/spark-tests/index.md b/docs/development/spark-tests/index.md
index e6415d58b..9be083fa1 100644
--- a/docs/development/spark-tests/index.md
+++ b/docs/development/spark-tests/index.md
@@ -1,6 +1,6 @@
 ---
 title: Spark Tests
-rank: 3
+rank: 4
 ---
 
 # Spark Tests
diff --git a/docs/development/spark-tests/spark-patch.md b/docs/development/spark-tests/spark-patch.md
index e878ae611..c8aa399c0 100644
--- a/docs/development/spark-tests/spark-patch.md
+++ b/docs/development/spark-tests/spark-patch.md
@@ -11,15 +11,15 @@ Here are the commands that can be helpful for this purpose.
 ```bash
 # Apply the patch.
 # You can now modify the Spark source code.
-git -C opt/spark apply ../../scripts/spark-tests/spark-4.1.1.patch
+git -C opt/spark apply ../../scripts/spark-tests/spark-4.2.0.patch
 
 # Update the Spark patch file with your local modification.
 git -C opt/spark add .
-git -C opt/spark diff --staged -p > scripts/spark-tests/spark-4.1.1.patch
+git -C opt/spark diff --staged -p > scripts/spark-tests/spark-4.2.0.patch
 
 # Revert the patch.
 git -C opt/spark reset
-git -C opt/spark apply -R ../../scripts/spark-tests/spark-4.1.1.patch
+git -C opt/spark apply -R ../../scripts/spark-tests/spark-4.2.0.patch
 ```
 
 However, note that we should keep the patch minimal.
diff --git a/docs/development/spark-tests/spark-setup.md b/docs/development/spark-tests/spark-setup.md
index ad0534a18..ac84091a6 100644
--- a/docs/development/spark-tests/spark-setup.md
+++ b/docs/development/spark-tests/spark-setup.md
@@ -15,7 +15,7 @@ git clone git@github.com:ibis-project/testing-data.git opt/ibis-testing-data
 
 ::: info
 
-You may use options such as `--branch v4.1.1 --depth 1` to get a shallow copy of the Spark repository. But you would need to take care of fetching other branches if you need to work with multiple Spark versions.
+You may use options such as `--branch v4.2.0 --depth 1` to get a shallow copy of the Spark repository. But you would need to take care of fetching other branches if you need to work with multiple Spark versions.
 
 :::
 
@@ -26,8 +26,8 @@ Python tests are also included in the patched package.
 You can choose to run any or all of the commands, depending on the Spark versions you want to test against.
 
 ```bash
-env SPARK_VERSION=4.1.1 scripts/spark-tests/build-pyspark.sh
-env SPARK_VERSION=3.5.7 scripts/spark-tests/build-pyspark.sh
+env SPARK_VERSION=4.2.0 scripts/spark-tests/build-pyspark.sh
+env SPARK_VERSION=3.5.9 scripts/spark-tests/build-pyspark.sh
 ```
 
 ::: info
diff --git a/docs/development/spark-tests/test-spark.md b/docs/development/spark-tests/test-spark.md
index 7c1aef42b..5075a80e4 100644
--- a/docs/development/spark-tests/test-spark.md
+++ b/docs/development/spark-tests/test-spark.md
@@ -18,8 +18,8 @@ We have developed custom scripts to run the tests using [pytest](https://docs.py
 Before running Spark tests, you need to install the [patched PySpark package](./spark-setup) to the `test-spark` matrix environments that supports testing against multiple Spark versions.
 
 ```bash
-hatch run test-spark.spark-4.1.1:install-pyspark
-hatch run test-spark.spark-3.5.7:install-pyspark
+hatch run test-spark.spark-4.2.0:install-pyspark
+hatch run test-spark.spark-3.5.9:install-pyspark
 ```
 
 You can choose to run any or all of the commands above, depending on whether you have built the patched PySpark package for the corresponding Spark version.
@@ -39,7 +39,7 @@ hatch run scripts/spark-tests/run-server.sh
 After running the Spark Connect server, start another terminal and use the following command to run the Spark tests.
 
 ```bash
-hatch run test-spark.spark-4.1.1:scripts/spark-tests/run-tests.sh
+hatch run test-spark.spark-4.2.0:scripts/spark-tests/run-tests.sh
 ```
 
 The command runs a default set of test suites for Spark Connect.
@@ -56,7 +56,7 @@ You can also use `PYTEST_` environment variables to customize the test execution
 For example, `PYTEST_ADDOPTS="-k "` can be used to run specific tests matching ``.
 
 ```bash
-hatch run test-spark.spark-4.1.1:env \
+hatch run test-spark.spark-4.2.0:env \
   TEST_RUN_NAME=selected \
   scripts/spark-tests/run-tests.sh \
   --pyargs pyspark.sql.tests.connect -v -k "test_sql"
@@ -73,7 +73,7 @@ As a comparison, you can run the tests against the original JVM-based Spark libr
 by setting the `SPARK_TESTING_REMOTE_PORT` environment variable to an empty string.
 
 ```bash
-hatch run test-spark.spark-4.1.1:env \
+hatch run test-spark.spark-4.2.0:env \
   SPARK_TESTING_REMOTE_PORT= \
   scripts/spark-tests/run-tests.sh
 ```
@@ -86,16 +86,16 @@ The steps above start the server in the `default` Hatch environment.
 There are a few PySpark UDF tests that would fail in this setup, since they import testing UDFs available only in the **patched** PySpark library (installed in the `test-spark` Hatch environment).
 There are also a few data-dependent tests that would fail, since the data files in the `python/test_support` directory are only available in the **patched** PySpark library.
 
-Moreover, when the server is started in the `default` environment which has the PySpark 4.1.1 library installed, the tests for PySpark 3.5.7 does not work.
+Moreover, when the server is started in the `default` environment which has the PySpark 4.2.0 library installed, the tests for PySpark 3.5.9 do not work.
 
 To use the same PySpark library for both the server and the tests, run the server and the tests in the same `test-spark` environment.
 
 ```bash
-hatch run test-spark.spark-3.5.7:scripts/spark-tests/run-server.sh
+hatch run test-spark.spark-3.5.9:scripts/spark-tests/run-server.sh
 ```
 
 ```bash
-hatch run test-spark.spark-3.5.7:scripts/spark-tests/run-tests.sh
+hatch run test-spark.spark-3.5.9:scripts/spark-tests/run-tests.sh
 ```
 
 However, running the server outside the `default` environment [pollutes the build cache](../recipes/reducing-build-time.md), so you may notice that the server takes longer to build and start.
diff --git a/docs/guide/catalog/glue.md b/docs/guide/catalog/glue.md
index 933fc6de9..fa3b7f9fc 100644
--- a/docs/guide/catalog/glue.md
+++ b/docs/guide/catalog/glue.md
@@ -7,11 +7,14 @@ rank: 4
 
 The AWS Glue catalog provider in Sail allows you to connect to an external [AWS Glue Data Catalog](https://aws.amazon.com/glue/).
 
-AWS Glue catalog can be configured using the following options:
+## Options
 
-- `type` (required): The string `glue`.
-- `name` (required): The name of the catalog.
-- `region` (optional): The AWS region (e.g., `us-east-1`). If not set, it uses the default region from the AWS credential provider chain.
+AWS Glue catalog can be configured using the following options.
+
+- `type` (required): The catalog provider. Set this option to `glue`.
+- `name` (required): The catalog name.
+- `catalog_id` (optional): The AWS Glue Data Catalog ID. If this option is not set, AWS uses the account ID associated with the active credentials.
+- `region` (optional): The AWS region. Set this option to a region such as `us-east-1`. If it is not set, AWS uses the default region from the AWS credential provider chain.
 - `endpoint_url` (optional): The custom endpoint URL.
 
 See [Common Options](./index.md#common-options) for caching configuration.
@@ -20,17 +23,32 @@ You can use any AWS credential provider supported by the AWS SDK to authenticate
 
 ## Examples
 
+This example uses the AWS credential provider chain with a catalog in `us-west-2`.
+
 ```bash
 export SAIL_CATALOG__LIST='[{type="glue", name="sail", region="us-west-2"}]'
+```
+
+This example selects an AWS Glue Data Catalog by ID.
+
+```bash
+export SAIL_CATALOG__LIST='[{type="glue", name="sail", catalog_id="123456789012", region="us-west-2"}]'
+```
+
+This example uses a custom endpoint, such as a LocalStack instance.
 
-# Using a custom endpoint (e.g., LocalStack)
+```bash
 export SAIL_CATALOG__LIST='[{type="glue", name="sail", region="us-east-1", endpoint_url="http://localhost:4566"}]'
+```
 
-# Enabling caching for database and table listings
+This example enables shared caching for database and table listings.
+
+```bash
 export SAIL_CATALOG__LIST='[{type="glue", name="sail", region="us-west-2", database_cache_type="global", database_cache_size=100, database_cache_ttl_secs=3600, table_cache_type="global", table_cache_size=1000, table_cache_ttl_secs=3600}]'
+```
+
+This example uses a session-scoped database cache, which is private to one session.
 
-# Configuring cache scope (global or session)
-# Global scope shares the cache across all sessions.
-# Session scope keeps the cache private to a single session.
+```bash
 export SAIL_CATALOG__LIST='[{type="glue", name="sail", region="us-west-2", database_cache_type="session"}]'
 ```
diff --git a/docs/guide/catalog/hms.md b/docs/guide/catalog/hms.md
index a4554af5c..f75e9acba 100644
--- a/docs/guide/catalog/hms.md
+++ b/docs/guide/catalog/hms.md
@@ -5,66 +5,47 @@ rank: 6
 
 # Hive Metastore
 
-The Hive Metastore catalog provider in Sail allows you to connect to an external Hive Metastore service over Thrift.
+The Hive Metastore (HMS) catalog provider connects Sail to an external Hive Metastore service over Thrift. It lets Sail use the databases, tables, and views whose metadata is stored in that service.
 
-## Support Status
+Sail supports plain Thrift connections, Kerberos-protected Thrift SASL connections, and high-availability endpoint lists. It supports flat database namespaces and resolves the provider and location recorded for existing HMS tables. See [Data Sources](../sources/) for the formats Sail can read and write.
 
-Sail's HMS integration is currently aimed at metadata interoperability with Apache Hive Metastore deployments.
+Sail does not support Hive ACID operations, including transaction heartbeats, locks, and write ID allocation. It also does not support delegation-token authentication.
 
-The following areas are supported:
+## Options
 
-- Plain HMS connections over Thrift.
-- Kerberos-secured HMS connections over Thrift SASL.
-- HMS high-availability URI lists with endpoint failover.
-- Flat database namespaces.
-- Database, table, and view metadata stored in HMS.
-- Generic Hive storage formats: `parquet`, `csv`, `textfile`, `json`, `orc`, `avro`, and `delta` with the alias `deltalake`.
+An HMS catalog can be configured using the following options.
 
-The following areas are not implemented yet:
-
-- Hive ACID or transactional HMS APIs such as transaction heartbeats, locks, or write ID allocation.
-- Iceberg-in-HMS behavior.
-- Delegation-token authentication.
-
-Hive Metastore can be configured using the following options:
-
-- `type` (required): The string `hive_metastore` or the alias `hms`.
-- `name` (required): The name of the catalog.
-- `uris` (required): A list of HMS endpoints. Each entry accepts either `host:port` or `thrift://host:port`. Entries may also include comma-separated endpoint lists.
+- `type` (required): The catalog provider. Set this option to `hive_metastore` or `hms`.
+- `name` (required): The catalog name.
+- `uris` (required): The HMS endpoints. Provide a list whose entries use either `host:port` or `thrift://host:port`. An entry can contain a comma-separated list of endpoints.
 - `thrift_transport` (optional): The Thrift transport mode. Valid values are `buffered` and `framed`. The default is `buffered`.
 - `auth` (optional): The HMS authentication mode. Valid values are `none` and `kerberos`. The default is `none`.
-- `kerberos_service_principal` (optional): Required when `auth = "kerberos"`. Use the HMS service principal in the form `service/_HOST@REALM`, for example `hive-metastore/_HOST@EXAMPLE.COM`.
-- `min_sasl_qop` (optional): Minimum Kerberos SASL QOP when `auth = "kerberos"`. Valid values are `auth`, `auth_int`, and `auth_conf`. The default is `auth`.
-- `connect_timeout_secs` (optional): Per-endpoint connect timeout in seconds. The default is `5`.
+- `kerberos_service_principal` (optional): The HMS service principal. Set this option when `auth = "kerberos"`. Use the form `service/_HOST@REALM`, such as `hive-metastore/_HOST@EXAMPLE.COM`.
+- `min_sasl_qop` (optional): The minimum Kerberos SASL quality of protection. Set this option when `auth = "kerberos"`. Valid values are `auth`, `auth_int`, and `auth_conf`. The default is `auth`.
+- `connect_timeout_secs` (optional): The connection timeout for each endpoint. Set this option in seconds. The default is `5`.
 
-See [Common Options](./index.md#common-options) for caching configuration.
+See [Common Options](./index.md#common-options) for options that configure caching.
 
-Failover behavior:
+## Endpoint Failover Behavior
 
-- Sail attempts endpoints in configured order.
-- New connections re-resolve DNS for the selected endpoint instead of pinning the initial startup address forever.
-- Retryable transport/Thrift failures rotate to the next endpoint.
-- A retried create or drop normalizes `AlreadyExists` and `NotFound` responses when the prior attempt likely succeeded but the response was lost.
-- Per-endpoint connect timeout defaults to `5s` and can be overridden with `connect_timeout_secs`.
+Sail tries endpoints in the order in which they are configured. It resolves the selected endpoint's DNS name for each new connection, so connections do not remain pinned to the address found at startup. When a retryable transport or Thrift error occurs, Sail moves to the next endpoint. If a connection fails after a create or drop request may have succeeded, Sail treats the resulting `AlreadyExists` or `NotFound` response as a successful retry.
 
 ## Kerberos Authentication
 
 ::: info
-Kerberos authentication for Hive Metastore is supported and uses the same operator model as Sail's HDFS support.
+Sail uses the same Kerberos operating model for Hive Metastore and HDFS.
 :::
 
 ### Prerequisites
 
-- A Kerberos-enabled Hive Metastore service.
-- A valid `krb5.conf` file on the Sail server host.
-- A valid Kerberos ticket cache for the Sail server process.
-- Kerberos runtime libraries on the Sail server host.
-  On Linux Sail loads `libgssapi_krb5.so.2` at runtime.
-  On macOS install Kerberos libraries, for example with `brew install krb5`.
+- The Hive Metastore service must be configured for Kerberos.
+- The Sail server host must have a valid `krb5.conf` file.
+- The Sail server process must have a valid Kerberos ticket cache.
+- The Sail server host must have Kerberos runtime libraries. On Linux, Sail loads `libgssapi_krb5.so.2` at runtime. On macOS, you can install them with `brew install krb5`.
 
 ### Starting the Sail Server
 
-Authenticate with Kerberos before starting the Sail server.
+Run `kinit` before starting the Sail server so that the server process can use its ticket cache.
 
 ```python
 import subprocess
@@ -72,9 +53,10 @@ from pysail.spark import SparkConnectServer
 
 # authenticate with Kerberos
 subprocess.run([
-    "kinit", "-kt",
+    "kinit",
+    "-kt",
     "/path/to/user.keytab",
-    "username@YOUR.REALM"
+    "username@YOUR.REALM",
 ], check=True)
 
 # start the Sail server
@@ -83,50 +65,47 @@ server.start(background=False)
 ```
 
 ::: tip
-The Sail server uses the process ticket cache created by `kinit`.
-
-If you run Sail in a distributed environment, each worker needs its own Kerberos credentials.
+In a distributed deployment, every worker needs its own Kerberos credentials.
 :::
 
-### Kerberos HMS Catalog Configuration
-
-When `auth = "kerberos"` is enabled, Sail expands `_HOST` in `kerberos_service_principal` from the hostname of the endpoint selected for that connection attempt.
+### Kerberos Catalog Configuration
 
-```bash
-export SAIL_CATALOG__LIST='[{type="hms", name="sail", uris=["hms1.internal:9083","thrift://hms2.internal:9083"], auth="kerberos", kerberos_service_principal="hive-metastore/_HOST@EXAMPLE.COM"}]'
-```
-
-### Security Guarantees
+When `auth = "kerberos"`, Sail replaces `_HOST` in `kerberos_service_principal` with the hostname of the endpoint selected for that connection attempt.
 
-- Downgrade fail-fast: if `min_sasl_qop` cannot be satisfied by the server-advertised SASL layers, connection setup fails immediately.
-- Session-wide protection: once a wrapped QOP (`auth_int` or `auth_conf`) is negotiated, every Thrift frame for that connection is wrapped/unwrapped through the Kerberos SASL security layer.
+Sail fails the connection when the server cannot meet `min_sasl_qop`. Once it negotiates `auth_int` or `auth_conf`, it protects every Thrift frame for that connection with the Kerberos SASL security layer.
 
-### Current Limitations
-
-- Sail uses an existing Kerberos ticket cache. It does not run `kinit` or manage keytabs internally.
-- Delegation-token authentication is not supported.
-- Transactional Hive Metastore APIs are not used yet. Sail currently targets metadata CRUD rather than Hive ACID write coordination.
+Sail uses the existing ticket cache and does not run `kinit` or manage keytabs. It does not use delegation tokens or transactional HMS APIs. The integration is intended for metadata operations rather than Hive ACID write coordination.
 
 ## Table Types
 
-Sail distinguishes between **managed** and **external** tables based on the `table_type` field stored in HMS metadata:
-
-- **Managed tables** (created without `LOCATION`, e.g. by Spark) report `Type: MANAGED` in `DESCRIBE EXTENDED`.
-- **External tables** (created with `LOCATION` or by Sail itself) report `Type: EXTERNAL`.
+HMS records whether a table is managed or external in its `table_type` metadata. For tables created by Spark, tables created without `LOCATION` appear as `MANAGED`, and tables created with `LOCATION` appear as `EXTERNAL`.
 
-For `DROP TABLE`, Sail uses metadata-only semantics for HMS and does **not** request physical data deletion via the HMS `delete_data` flag, regardless of managed vs external type.
+Sail always creates tables as external by marking them as `EXTERNAL` and setting `table_type` to `EXTERNAL_TABLE`. For tables created by other engines, Sail reports the type stored in HMS.
 
-Sail always creates its own tables as external (`EXTERNAL=TRUE`, `table_type = EXTERNAL_TABLE`). When reading tables created by other engines (e.g. Spark), Sail reflects the type recorded in HMS.
+When Sail drops an HMS table, it removes only the metadata. It does not ask HMS to delete the table's data, regardless of the table type.
 
 ## Examples
 
+The following example configures a single unencrypted HMS endpoint.
+
 ```bash
 export SAIL_CATALOG__LIST='[{type="hive_metastore", name="sail", uris=["127.0.0.1:9083"]}]'
+```
+
+This example uses two endpoints, a framed Thrift transport, and a ten-second connection timeout.
 
+```bash
 export SAIL_CATALOG__LIST='[{type="hms", name="sail", uris=["hms1.internal:9083","hms2.internal:9083"], thrift_transport="framed", connect_timeout_secs=10}]'
+```
 
+This example connects to an HMS service with Kerberos authentication and requires integrity protection for the connection.
+
+```bash
 export SAIL_CATALOG__LIST='[{type="hms", name="sail", uris=["hms.internal:9083"], auth="kerberos", kerberos_service_principal="hive-metastore/_HOST@EXAMPLE.COM", min_sasl_qop="auth_int", thrift_transport="framed"}]'
+```
 
-# Enabling caching for database and table listings
+This example enables shared caching for database and table listings.
+
+```bash
 export SAIL_CATALOG__LIST='[{type="hms", name="sail", uris=["127.0.0.1:9083"], database_cache_type="global", database_cache_ttl_secs=3600, table_cache_type="global", table_cache_size=1000}]'
 ```
diff --git a/docs/guide/catalog/iceberg-rest.md b/docs/guide/catalog/iceberg-rest.md
index 06b7dd5d3..c39cb1949 100644
--- a/docs/guide/catalog/iceberg-rest.md
+++ b/docs/guide/catalog/iceberg-rest.md
@@ -7,19 +7,19 @@ rank: 2
 
 The Iceberg REST catalog provider in Sail allows you to connect to an external catalog that exposes the [Iceberg REST Catalog API](https://iceberg.apache.org/rest-catalog-spec/).
 
-An Iceberg REST catalog can be configured using the following options:
+## Options
 
-- `type` (required): The string `iceberg-rest`.
-- `name` (required): The name of the catalog.
+An Iceberg REST catalog can be configured using the following options.
+
+- `type` (required): The catalog provider. Set this option to `iceberg-rest`.
+- `name` (required): The catalog name.
 - `uri` (required): The base URI of the Iceberg REST catalog server.
-- `warehouse` (optional): The warehouse location for the catalog.
-- `prefix` (optional): The prefix for all catalog API endpoints.
-- `namespace_separator` (optional): A client-side fallback for the Iceberg REST `namespace-separator` catalog property.
-  Use the separator string the REST server expects, such as `::` or `/`.
-  Sail also accepts URL-encoded values returned by REST servers, such as `%3A%3A` for `::`.
-  If unset or empty, Sail uses the Iceberg REST default unit separator.
+- `warehouse` (optional): The catalog warehouse location.
+- `prefix` (optional): The prefix for catalog API endpoints.
+- `namespace_separator` (optional): The client-side fallback for the Iceberg REST `namespace-separator` catalog property. Use the separator that the REST server expects, such as `::` or `/`. Sail also accepts URL-encoded values returned by REST servers, such as `%3A%3A` for `::`. If this option is not set or is empty, Sail uses the Iceberg REST default unit separator.
 - `oauth_access_token` (optional): The OAuth 2.0 access token.
-- `bearer_access_token` (optional): The bearer token for authentication.
+- `bearer_access_token` (optional): The bearer token for authentication. This option takes precedence over `oauth_access_token`.
+- `bearer_access_token_file` (optional): The path to a file that holds the bearer token. This option takes precedence over `bearer_access_token` and `oauth_access_token`. Sail reads the token from this file for every request, so it picks up a rotated token, such as a kubelet-projected service account token, without restarting the server. If a request is rejected with `401 Unauthorized`, Sail reloads the file and retries the request once. The file must contain the raw token without the `Bearer ` prefix for the HTTP `Authorization` header. Surrounding whitespace in the file is trimmed. Empty or unreadable files produce an error rather than falling back to other options.
 
 See [Common Options](./index.md#common-options) for caching configuration.
 
@@ -27,23 +27,40 @@ See [Common Options](./index.md#common-options) for caching configuration.
 
 Sail calls `GET /v1/config` before catalog operations. The final Iceberg REST catalog configuration is merged in this order:
 
-1. Server `defaults`
-2. Sail client configuration
-3. Server `overrides`
+1. Server `defaults`.
+2. Sail client configuration.
+3. Server `overrides`.
 
-Server overrides take precedence over matching values configured in Sail. For example, if the REST server returns `namespace-separator` in `overrides`, Sail uses that value to encode multipart namespaces even when `namespace_separator` is configured locally. Configuring `namespace_separator` in Sail does not configure the REST server; the server must already decode the same separator, usually by advertising it from `/v1/config`.
+Server overrides take precedence over matching values configured in Sail. For example, if the REST server returns `namespace-separator` in `overrides`, Sail uses that value to encode multipart namespaces even when `namespace_separator` is configured locally. Configuring `namespace_separator` in Sail does not configure the REST server. The server must already decode the same separator, usually by advertising it from `/v1/config`.
 
 ## Examples
 
+This example configures an Iceberg REST catalog without authentication.
+
 ```bash
 export SAIL_CATALOG__LIST='[{type="iceberg-rest", name="sail", uri="https://catalog.example.com"}]'
+```
+
+This example uses an OAuth access token.
 
-# OAuth authentication
+```bash
 export SAIL_CATALOG__LIST='[{type="iceberg-rest", name="sail", uri="https://catalog.example.com", warehouse="s3://data/warehouse", oauth_access_token="..."}]'
+```
 
-# Bearer token authentication
+This example uses a bearer token.
+
+```bash
 export SAIL_CATALOG__LIST='[{type="iceberg-rest", name="sail", uri="https://catalog.example.com", warehouse="s3://data/warehouse", bearer_access_token="..."}]'
+```
+
+This example reads a bearer token from a file, such as a kubelet-projected service account token.
 
-# Client-side namespace separator fallback
+```bash
+export SAIL_CATALOG__LIST='[{type="iceberg-rest", name="sail", uri="https://catalog.example.com", warehouse="s3://data/warehouse", bearer_access_token_file="/var/run/secrets/tokens/catalog-token"}]'
+```
+
+This example configures a client-side namespace separator fallback.
+
+```bash
 export SAIL_CATALOG__LIST='[{type="iceberg-rest", name="sail", uri="https://catalog.example.com", namespace_separator="::"}]'
 ```
diff --git a/docs/guide/catalog/index.md b/docs/guide/catalog/index.md
index 74b004a5a..0d470a07e 100644
--- a/docs/guide/catalog/index.md
+++ b/docs/guide/catalog/index.md
@@ -43,15 +43,15 @@ The next few pages describe the different catalog providers supported by Sail an
 
 All remote catalog providers (excluding the [Memory catalog](./memory)) support the following common options for caching database and table listings. This is particularly useful for reducing the number of requests to external services.
 
-- `database_cache_type` (optional): The scope of the database listing cache. Valid values are `none`, `global` (shared across all sessions), and `session` (private to the current session). (default: `none`).
-- `database_cache_size` (optional): Maximum number of entries in the database list cache. `0` = unbounded.
-- `database_cache_ttl_secs` (optional): Time-to-live in seconds for cached database listings. `0` = no expiry.
-- `table_cache_type` (optional): The scope of the table listing cache. Valid values are `none`, `global`, and `session`. (default: `none`).
-- `table_cache_size` (optional): Maximum number of entries in the table list cache. `0` = unbounded.
-- `table_cache_ttl_secs` (optional): Time-to-live in seconds for cached table listings. `0` = no expiry.
-- `view_cache_type` (optional): The scope of the view listing cache. Valid values are `none`, `global`, and `session`. (default: `none`).
-- `view_cache_size` (optional): Maximum number of entries in the view list cache. `0` = unbounded.
-- `view_cache_ttl_secs` (optional): Time-to-live in seconds for cached view listings. `0` = no expiry.
+- `database_cache_type` (optional): The scope of the database listing cache. Valid values are `none`, `global`, which shares the cache across sessions, and `session`, which keeps the cache private to one session. The default is `none`.
+- `database_cache_size` (optional): The maximum number of entries in the database listing cache. Set this option to `0` for an unbounded cache.
+- `database_cache_ttl_secs` (optional): The time-to-live for cached database listings. Set this option in seconds. Set it to `0` to disable expiration.
+- `table_cache_type` (optional): The scope of the table listing cache. Valid values are `none`, `global`, and `session`. The default is `none`.
+- `table_cache_size` (optional): The maximum number of entries in the table listing cache. Set this option to `0` for an unbounded cache.
+- `table_cache_ttl_secs` (optional): The time-to-live for cached table listings. Set this option in seconds. Set it to `0` to disable expiration.
+- `view_cache_type` (optional): The scope of the view listing cache. Valid values are `none`, `global`, and `session`. The default is `none`.
+- `view_cache_size` (optional): The maximum number of entries in the view listing cache. Set this option to `0` for an unbounded cache.
+- `view_cache_ttl_secs` (optional): The time-to-live for cached view listings. Set this option in seconds. Set it to `0` to disable expiration.
 
 The cache is automatically invalidated when a write operation (like `CREATE TABLE` or `DROP DATABASE`) is performed through Sail.
 
diff --git a/docs/guide/catalog/memory.md b/docs/guide/catalog/memory.md
index 0802c8567..f41942e2d 100644
--- a/docs/guide/catalog/memory.md
+++ b/docs/guide/catalog/memory.md
@@ -7,11 +7,13 @@ rank: 1
 
 The memory catalog provider in Sail stores table metadata in memory for the duration of your session. This is a useful default choice if you do not need persistent table metadata across sessions.
 
+## Options
+
 A memory catalog can be configured using the following options.
 
-- `type` (required): The string `memory`.
-- `name` (required): The name of the catalog.
-- `initial_database` (required): The initial database namespace as an array of strings.
+- `type` (required): The catalog provider. Set this option to `memory`.
+- `name` (required): The catalog name.
+- `initial_database` (required): The initial database namespace. Provide an array of strings.
 - `initial_database_comment` (optional): The comment for the initial database.
 
 ## Examples
diff --git a/docs/guide/catalog/onelake.md b/docs/guide/catalog/onelake.md
index 8106cc8c2..e89ace75a 100644
--- a/docs/guide/catalog/onelake.md
+++ b/docs/guide/catalog/onelake.md
@@ -7,32 +7,107 @@ rank: 5
 
 The OneLake catalog provider in Sail allows you to connect to [Microsoft Fabric OneLake](https://learn.microsoft.com/en-us/fabric/onelake/onelake-overview).
 
-OneLake catalog can be configured using the following options:
+## Options
 
-- `type` (required): The string `onelake`.
-- `name` (required): The name of the catalog.
+OneLake catalog can be configured using the following options.
+
+- `type` (required): The catalog provider. Set this option to `onelake`.
+- `name` (required): The catalog name.
 - `url` (required): The OneLake item location.
+- `api` (optional): The OneLake catalog API. Use `delta` for the Unity Catalog endpoint or `iceberg` for the Iceberg REST catalog endpoint. The default is `iceberg`.
 - `bearer_token` (optional): The bearer token for authentication.
 
 See [Common Options](./index.md#common-options) for caching configuration.
 
 The `url` should be in the format .
 
-If `bearer_token` is not provided, Sail will attempt to find credentials from the following sources in order:
+If `bearer_token` is not specified, Sail creates credentials from environment variables, similar to how authentication is handled in [Azure storage](../storage/azure.md).
+These credentials acquire Microsoft Entra tokens for the Azure Storage audience required by OneLake.
+
+The following environment variables are supported:
+
+- `AZURE_STORAGE_TOKEN`, `AZURE_ACCESS_TOKEN`
+
+  The bearer token for authorizing requests.
+
+- `AZURE_STORAGE_CLIENT_ID`, `AZURE_CLIENT_ID`
+
+  The service principal client ID for authorizing requests. This can also identify a user-assigned managed identity.
+
+- `AZURE_STORAGE_CLIENT_SECRET`, `AZURE_CLIENT_SECRET`
+
+  The service principal client secret for authorizing requests.
+
+- `AZURE_STORAGE_TENANT_ID`, `AZURE_STORAGE_AUTHORITY_ID`, `AZURE_TENANT_ID`, `AZURE_AUTHORITY_ID`
+
+  The tenant ID used in OAuth flows.
+
+- `AZURE_STORAGE_AUTHORITY_HOST`, `AZURE_AUTHORITY_HOST`
+
+  The authority host used in OAuth flows.
+
+- `AZURE_MSI_ENDPOINT`, `AZURE_IDENTITY_ENDPOINT`
+
+  The endpoint for acquiring a managed identity token.
+
+- `AZURE_OBJECT_ID`
+
+  The object ID for use with managed identity authentication.
+
+- `AZURE_MSI_RESOURCE_ID`
+
+  The MSI resource ID for use with managed identity authentication.
+
+- `AZURE_FEDERATED_TOKEN_FILE`
+
+  The file containing a token for Azure AD workload identity federation.
+
+- `AZURE_USE_AZURE_CLI`
 
-1. The `AZURE_STORAGE_TOKEN` environment variable.
-2. The `AZURE_ACCESS_TOKEN` environment variable.
-3. Azure CLI (`az account get-access-token`).
+  Whether to use Azure CLI for acquiring an access token.
+
+Tokens are refreshed automatically before expiry.
+
+::: warning
+Azure account keys and SAS tokens are not supported for OneLake catalog authentication. OneLake requires Microsoft Entra bearer tokens.
+:::
 
 ## Examples
 
+This example uses the OneLake Unity Catalog endpoint.
+
 ```bash
-export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse"}]'
+export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse", api="delta"}]'
+```
+
+This example uses the OneLake Iceberg REST catalog endpoint.
 
-# Bearer token authentication
+```bash
+export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse", api="iceberg"}]'
+```
+
+This example supplies a bearer token directly.
+
+```bash
 export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse", bearer_token="..."}]'
 ```
 
+This example authenticates with a service principal and tenant ID.
+
+```bash
+export AZURE_TENANT_ID="..."
+export AZURE_CLIENT_ID="..."
+export AZURE_CLIENT_SECRET="..."
+export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse"}]'
+```
+
+This example uses Azure CLI authentication.
+
+```bash
+export AZURE_USE_AZURE_CLI=true
+export SAIL_CATALOG__LIST='[{type="onelake", name="fabric", url="workspace/lakehouse.Lakehouse"}]'
+```
+
 
diff --git a/docs/guide/catalog/unity.md b/docs/guide/catalog/unity.md
index 3aecf7f84..1f66b27d6 100644
--- a/docs/guide/catalog/unity.md
+++ b/docs/guide/catalog/unity.md
@@ -8,15 +8,15 @@ rank: 3
 The Unity Catalog provider in Sail allows you to connect to an external catalog that exposes
 the [Unity Catalog API](https://docs.unitycatalog.io/).
 
-Unity Catalog can be configured using the following options:
+## Options
 
-- `type` (required): The string `unity`.
-- `name` (required): The name of the catalog.
-- `uri` (optional): The base URI of the Unity Catalog API. Defaults to `http://localhost:8080/api/2.1/unity-catalog`.
-- `default_catalog` (optional): The default catalog name to use. Defaults to `unity`.
-- `token` (optional): The authentication token to use when connecting to the Unity Catalog API. Defaults to empty (no
-  authentication). Please refer
-  to [Unity Catalog Authentication and Authorization](https://docs.unitycatalog.io/server/auth/) for more information.
+Unity Catalog can be configured using the following options.
+
+- `type` (required): The catalog provider. Set this option to `unity`.
+- `name` (required): The catalog name.
+- `uri` (optional): The base URI of the Unity Catalog API. The default is `http://localhost:8080/api/2.1/unity-catalog`.
+- `default_catalog` (optional): The default catalog name. The default is `unity`.
+- `token` (optional): The authentication token for the Unity Catalog API. The default is an empty value, which disables authentication. See [Unity Catalog Authentication and Authorization](https://docs.unitycatalog.io/server/auth/) for more information.
 
 See [Common Options](./index.md#common-options) for caching configuration.
 
@@ -28,9 +28,9 @@ The environment variables only allow configuring one Unity Catalog provider inst
 
 Sail automatically loads Unity Catalog configuration from environment variables with the following prefixes:
 
-- `DATABRICKS_*` - Databricks-specific configuration
-- `UNITY_*` - Unity Catalog configuration
-- `UC_*` - Short-form Unity Catalog configuration
+- `DATABRICKS_*`: Databricks-specific configuration.
+- `UNITY_*`: Unity Catalog configuration.
+- `UC_*`: Short-form Unity Catalog configuration.
 
 The following table lists the supported configuration keys and their corresponding environment variable names:
 
diff --git a/docs/guide/dataframe/features.md b/docs/guide/dataframe/features.md
index dfff65028..81a90931e 100644
--- a/docs/guide/dataframe/features.md
+++ b/docs/guide/dataframe/features.md
@@ -26,6 +26,7 @@ The list covers the most common use cases of the DataFrame API and is not meant
 | NA functions (`DataFrame.na`)                                                                            | :white_check_mark: |
 | Statistics functions (`DataFrame.stat`)                                                                  | :construction:     |
 | View management (e.g. `DataFrame.createOrReplaceTempView()`)                                             | :white_check_mark: |
+| Checkpoints (`DataFrame.checkpoint()`)                                                                   | :white_check_mark: |
 | RDD Access (`DataFrame.rdd`)                                                                             | :x:                |
 | PySpark UDFs                                                                                             | :white_check_mark: |
 | PySpark UDTFs                                                                                            | :white_check_mark: |
diff --git a/docs/guide/deployment/_common/python-version.md b/docs/guide/deployment/_common/python-version.md
new file mode 100644
index 000000000..2a3079dc8
--- /dev/null
+++ b/docs/guide/deployment/_common/python-version.md
@@ -0,0 +1,5 @@
+::: warning
+
+Please make sure to choose a Python version that is compatible with the PySpark version you are using. For example, PySpark 3.5 supports Python 3.8, 3.9, 3.10, and 3.11, so later versions of Python may have compatibility issues.
+
+:::
diff --git a/docs/guide/deployment/docker-images/production.md b/docs/guide/deployment/docker-images/production.md
index e3a6c26d4..1e37ffc4b 100644
--- a/docs/guide/deployment/docker-images/production.md
+++ b/docs/guide/deployment/docker-images/production.md
@@ -17,6 +17,8 @@ In an empty directory, create a `Dockerfile` with the following content.
 
 :::
 
+
+
 In the same directory, run the following command with the desired release tag or branch name to build the Docker image.
 
 ```bash-vue
diff --git a/docs/guide/deployment/docker-images/quick-start.md b/docs/guide/deployment/docker-images/quick-start.md
index 4749a1880..08f47405a 100644
--- a/docs/guide/deployment/docker-images/quick-start.md
+++ b/docs/guide/deployment/docker-images/quick-start.md
@@ -22,6 +22,8 @@ In an empty directory, create a `Dockerfile` with the following content.
 
 :::
 
+
+
 In the same directory, run the following command to build the Docker image.
 
 ```bash-vue
diff --git a/docs/guide/deployment/kubernetes.md b/docs/guide/deployment/kubernetes.md
index 53a7c450c..75e7e2a41 100644
--- a/docs/guide/deployment/kubernetes.md
+++ b/docs/guide/deployment/kubernetes.md
@@ -102,6 +102,12 @@ The Sail Spark Connect server runs as a Kubernetes deployment, and the gRPC port
 kubectl apply -f k8s/sail.yaml
 ```
 
+::: info
+
+The example sets the `SAIL_MODE` environment variable to `kubernetes-cluster` for distributed execution. You can still run Sail in local mode in Kubernetes by setting `SAIL_MODE` to `local` or removing the environment variable. In local mode, the Sail server runs in a single pod and does not launch worker pods.
+
+:::
+
 ## Overriding the Default Pod Spec
 
 By default, the worker pod spec is created programmatically. If the `kubernetes.worker_pod_template` configuration option is provided, it is merged into
diff --git a/docs/guide/integrations/langchain.md b/docs/guide/integrations/langchain.md
new file mode 100644
index 000000000..4ab0b077a
--- /dev/null
+++ b/docs/guide/integrations/langchain.md
@@ -0,0 +1,343 @@
+---
+title: LangChain
+rank: 6
+---
+
+# LangChain
+
+[LangChain](https://www.langchain.com/) provides tools for building applications and agents with language models.
+The [`langchain-sail`](https://github.com/lakehq/langchain-sail) package connects LangChain agents to Sail through Spark Connect.
+Agents can list tables, inspect schemas and sample rows, check SQL, and execute queries.
+
+## Overview
+
+`SailSQLToolkit` contains four tools for table discovery, schema inspection, query checking, and query execution.
+Each tool uses a `SailSQL` connection to Sail over Spark Connect.
+
+## Installation
+
+Install `langchain-sail` with the lightweight Spark Connect client.
+The following commands also install LangChain and the OpenAI integration used in this guide.
+
+::: code-group
+
+```bash [pip]
+pip install "langchain-sail[spark]" langchain langchain-openai
+```
+
+```bash [uv]
+uv add "langchain-sail[spark]" langchain langchain-openai
+```
+
+:::
+
+The `spark` extra installs `pyspark-client==4.2.0` without the JVM components included in the full PySpark distribution.
+
+Applications that require the full PySpark distribution should install it separately instead of using the `spark` extra.
+
+::: code-group
+
+```bash [pip]
+pip install langchain-sail "pyspark[connect]==4.2.0" langchain langchain-openai
+```
+
+```bash [uv]
+uv add langchain-sail "pyspark[connect]==4.2.0" langchain langchain-openai
+```
+
+:::
+
+To run a local Sail server for development or testing, also install `pysail`.
+
+::: code-group
+
+```bash [pip]
+pip install pysail
+```
+
+```bash [uv]
+uv add pysail
+```
+
+:::
+
+## Starting or Connecting to Sail
+
+### Connecting to an Existing Sail Server
+
+Connect a Spark session to the Sail server.
+
+```python
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder.remote("sc://your-sail-host:50051").getOrCreate()
+```
+
+See [Deployment](/guide/deployment/) for ways to run Sail outside the application.
+
+### Starting Sail Locally
+
+For local development, start Sail in the application process and connect a Spark session to its listening address.
+
+```python
+from pysail.spark import SparkConnectServer
+from pyspark.sql import SparkSession
+
+server = SparkConnectServer()
+server.start()
+_, port = server.listening_address
+
+spark = SparkSession.builder.remote(f"sc://localhost:{port}").getOrCreate()
+```
+
+Keep the server running while the integration is in use.
+Stop the Spark session with `spark.stop()` before calling `server.stop()`.
+
+### Preparing the Example Data
+
+A local Sail server starts without tables.
+Create two temporary views for the remaining examples.
+
+```python
+spark.createDataFrame(
+    [(1, "Alice"), (2, "Bob")],
+    ["customer_id", "name"],
+).createOrReplaceTempView("customers")
+
+spark.createDataFrame(
+    [(101, 1), (102, 1), (103, 2)],
+    ["order_id", "customer_id"],
+).createOrReplaceTempView("orders")
+```
+
+Temporary views exist only for the current Spark Connect session and do not modify persistent data.
+
+## Creating the Toolkit
+
+Set the OpenAI API key in your environment.
+
+```bash
+export OPENAI_API_KEY="your-api-key"
+```
+
+`SailSQLToolkit` requires a tool-capable chat model and a `SailSQL` instance.
+The query-checking tool uses the model.
+The `spark` variable refers to the Spark Connect session created above.
+
+```python
+from langchain.chat_models import init_chat_model
+from langchain_sail import SailSQL, SailSQLToolkit
+
+model = init_chat_model("openai:gpt-5.5")
+
+sail_sql = SailSQL(
+    spark_session=spark,
+    sample_rows_in_table_info=3,
+)
+
+toolkit = SailSQLToolkit(db=sail_sql, llm=model)
+tools = toolkit.get_tools()
+
+print([tool.name for tool in tools])
+```
+
+Output:
+
+```text
+['query_sql_db', 'schema_sql_db', 'list_tables_sql_db', 'query_checker_sql_db']
+```
+
+Pass existing table names through `include_tables` or `ignore_tables` to control which tables the toolkit can access.
+Use `sample_rows_in_table_info` to set the number of sample rows included with table schema information.
+
+## Available Tools
+
+`SailSQLToolkit.get_tools()` returns the following tools:
+
+| Tool                   | Input                                 | Output                                             | Purpose                                                  |
+| ---------------------- | ------------------------------------- | -------------------------------------------------- | -------------------------------------------------------- |
+| `list_tables_sql_db`   | An empty string                       | A comma-separated list of table names              | Lists the tables available to the toolkit                |
+| `schema_sql_db`        | A comma-separated list of table names | Table schemas and sample rows, or an error message | Inspects tables before constructing a query              |
+| `query_checker_sql_db` | A SQL query                           | The checked SQL query                              | Uses the language model to check for common SQL mistakes |
+| `query_sql_db`         | A SQL query                           | Query results or an error message                  | Executes the query against Sail                          |
+
+## Invoking Tools Directly
+
+Select tools by name to invoke them without an agent.
+These examples use the `customers` and `orders` temporary views created above.
+
+```python
+tools_by_name = {tool.name: tool for tool in tools}
+```
+
+### Listing Tables
+
+Invoke the table-listing tool with an empty string.
+
+```python
+tables = tools_by_name["list_tables_sql_db"].invoke("")
+print(tables)
+```
+
+Output:
+
+```text
+customers, orders
+```
+
+### Inspecting Table Schemas
+
+Provide table names as a comma-separated string.
+
+```python
+schema = tools_by_name["schema_sql_db"].invoke("customers, orders")
+print(schema)
+```
+
+Output:
+
+```text
+CREATE TABLE customers (customer_id bigint, name string);
+
+/*
+3 rows from customers table:
+customer_id	name
+1	Alice
+2	Bob
+*/
+
+CREATE TABLE orders (order_id bigint, customer_id bigint);
+
+/*
+3 rows from orders table:
+order_id	customer_id
+101	1
+102	1
+103	2
+*/
+```
+
+### Checking a Query
+
+Check model-generated SQL before execution.
+
+```python
+query = """SELECT c.name, COUNT(o.order_id) AS order_count
+FROM customers AS c
+LEFT JOIN orders AS o ON c.customer_id = o.customer_id
+GROUP BY c.name
+ORDER BY order_count DESC"""
+
+checked_query = tools_by_name["query_checker_sql_db"].invoke(query)
+print(checked_query)
+```
+
+Output:
+
+```sql
+SELECT c.name, COUNT(o.order_id) AS order_count
+FROM customers AS c
+LEFT JOIN orders AS o ON c.customer_id = o.customer_id
+GROUP BY c.name
+ORDER BY order_count DESC
+```
+
+### Executing the Query
+
+Pass the checked SQL to the query tool.
+
+```python
+result = tools_by_name["query_sql_db"].invoke(checked_query)
+print(result)
+```
+
+Output:
+
+```text
+[('Alice', '2'), ('Bob', '1')]
+```
+
+## Using the Toolkit with an Agent
+
+LangChain's `create_agent()` API combines the model, tools, and a system prompt.
+The prompt requires the agent to inspect available data and construct read-only queries.
+
+```python
+from langchain.agents import create_agent
+
+system_prompt = """
+You are an agent designed to interact with Sail using Spark SQL.
+Given a question, create a syntactically correct Spark SQL query, execute it,
+and use the result to answer the question. Unless the user requests a specific
+number of results, limit the query to at most 5 rows.
+
+Only select the columns needed to answer the question. Do not issue statements
+that modify data or schema, including INSERT, UPDATE, DELETE, DROP, or ALTER.
+
+Always list the available tables first. Then inspect the schemas of the relevant
+tables. Always check a query with query_checker_sql_db before executing it with
+query_sql_db. If query execution returns an error, revise the query and try again.
+"""
+
+agent = create_agent(
+    model,
+    tools,
+    system_prompt=system_prompt,
+)
+```
+
+Send a user message to the agent.
+
+```python
+response = agent.invoke(
+    {
+        "messages": [
+            {
+                "role": "user",
+                "content": "Which customer has placed the most orders?",
+            }
+        ]
+    }
+)
+
+print(response["messages"][-1].text)
+```
+
+Output:
+
+```text
+Alice has placed the most orders, with 2 orders.
+```
+
+`create_sail_sql_agent()` remains available for applications that use LangChain's classic `AgentExecutor` API.
+Use `create_agent()` for new applications.
+
+## Safety Considerations
+
+::: warning
+`query_sql_db` can execute arbitrary model-generated SQL against the connected Sail environment.
+The query checker asks a language model to review SQL for common mistakes, but it does not enforce access control and is not a security boundary.
+:::
+
+When using the toolkit:
+
+- Use a read-only, least-privilege role.
+- Limit access to required catalogs and schemas. Use `include_tables` to allowlist tables where possible.
+- Set server-side statement timeouts, resource limits, and concurrency limits.
+- Log generated queries and monitor their resource usage.
+- Require human approval before `query_sql_db` executes queries in sensitive environments.
+
+Client-side timeouts do not guarantee that a running query is cancelled on the server.
+
+## API Reference and Links
+
+`langchain-sail` exports `SailSQL`, `SailSQLToolkit`, the individual Sail SQL tool classes, and `create_sail_sql_agent()`.
+
+- [`langchain-sail` source code](https://github.com/lakehq/langchain-sail)
+- [`langchain-sail` on PyPI](https://pypi.org/project/langchain-sail/)
+- [Sail Python API reference](/reference/python/)
+- [Deploying Sail](/guide/deployment/)
+- [LangChain tools documentation](https://docs.langchain.com/oss/python/langchain/tools)
+- [LangChain agents documentation](https://docs.langchain.com/oss/python/langchain/agents)
+- [LangChain SQL agent guide](https://docs.langchain.com/oss/python/langchain/sql-agent)
+- [LangChain human-in-the-loop documentation](https://docs.langchain.com/oss/python/langchain/human-in-the-loop)
diff --git a/docs/guide/integrations/mcp-server.md b/docs/guide/integrations/mcp-server.md
index 9dbba6bf8..4ecd74fd2 100644
--- a/docs/guide/integrations/mcp-server.md
+++ b/docs/guide/integrations/mcp-server.md
@@ -36,7 +36,7 @@ If you have installed Sail via `pip` in a Python virtual environment, the path i
 Please note that you must install the PySail library with MCP dependencies.
 
 ```bash-vue
-pip install "pysail[mcp]=={{ libVersion }}" "pyspark-client==4.1.1"
+pip install "pysail[mcp]=={{ libVersion }}" "pyspark-client==4.2.0"
 ```
 
 ::: info
diff --git a/docs/guide/sources/delta/examples.md b/docs/guide/sources/delta/examples.md
index ae4b48897..b0e83fd02 100644
--- a/docs/guide/sources/delta/examples.md
+++ b/docs/guide/sources/delta/examples.md
@@ -100,8 +100,6 @@ df = spark.read.format("delta").option("versionAsOf", "0").load(path)
 df = spark.read.format("delta").option("timestampAsOf", "2025-01-02T03:04:05.678").load(path)
 ```
 
-Time travel is not available for Spark SQL in Sail yet, but we plan to support it soon.
-
 ## Column Mapping
 
 You can write Delta tables with column mapping enabled. The supported column mapping modes are `name` and `id`. You must write to a new Delta table to enable column mapping.
@@ -112,8 +110,3 @@ df.write.format("delta").option("columnMappingMode", "id").save(path)
 ```
 
 Existing Delta tables with column mapping can be read as usual.
-
-## More Features
-
-We will continue adding more examples for advanced Delta Lake features as they become available in Sail.
-For questions, reach out on [Slack](https://lakesail.com/slack) or [GitHub Discussions](https://github.com/lakehq/sail/discussions).
diff --git a/docs/guide/sources/iceberg/examples.md b/docs/guide/sources/iceberg/examples.md
index 3571fa744..9770cbce7 100644
--- a/docs/guide/sources/iceberg/examples.md
+++ b/docs/guide/sources/iceberg/examples.md
@@ -82,10 +82,3 @@ df = spark.read.format("iceberg").option("snapshotId", "123").load(path)
 df = spark.read.format("iceberg").option("timestampAsOf", "2025-01-02T03:04:05.678").load(path)
 df = spark.read.format("iceberg").option("branch", "main").load(path)
 ```
-
-Time travel is not available for Spark SQL in Sail yet, but we plan to support it soon.
-
-## More Features
-
-We will continue adding more examples for advanced Iceberg features as they become available in Sail.
-For questions, reach out on [Slack](https://lakesail.com/slack) or [GitHub Discussions](https://github.com/lakehq/sail/discussions).
diff --git a/docs/guide/storage/azure.md b/docs/guide/storage/azure.md
index 35dcc41dd..f5204bd98 100644
--- a/docs/guide/storage/azure.md
+++ b/docs/guide/storage/azure.md
@@ -45,7 +45,7 @@ Sail supports the following URI formats for Azure storage services:
       
  • 
         
         (Hadoop driver convention with Microsoft Fabric)
       
    • 
diff --git a/docs/guide/storage/hdfs.md b/docs/guide/storage/hdfs.md
index ccc15d6b0..75d04e69c 100644
--- a/docs/guide/storage/hdfs.md
+++ b/docs/guide/storage/hdfs.md
@@ -40,9 +40,10 @@ from pysail.spark import SparkConnectServer
 
 # authenticate with Kerberos
 subprocess.run([
-    "kinit", "-kt",
+    "kinit",
+    "-kt",
     "/path/to/user.keytab",
-    "username@YOUR.REALM"
+    "username@YOUR.REALM",
 ], check=True)
 
 # start the Sail server
diff --git a/docs/guide/storage/index.md b/docs/guide/storage/index.md
index 605ecd3df..cb3a25082 100644
--- a/docs/guide/storage/index.md
+++ b/docs/guide/storage/index.md
@@ -45,6 +45,7 @@ Here is a summary of the supported (:white_check_mark:) and unsupported (:x:) st
 | [Memory](./memory)                        | :white_check_mark: | :white_check_mark: |
 | [AWS S3](./s3)                            | :white_check_mark: | :white_check_mark: |
 | [Cloudflare R2](./s3)                     | :white_check_mark: | :white_check_mark: |
+| [Aliyun OSS](./s3#aliyun-oss)             | :white_check_mark: | :white_check_mark: |
 | [Azure Data Lake Storage (ADLS)](./azure) | :white_check_mark: | :white_check_mark: |
 | [Azure Blob Storage](./azure)             | :white_check_mark: | :white_check_mark: |
 | [Google Cloud Storage](./gcs)             | :white_check_mark: | :white_check_mark: |
@@ -57,7 +58,7 @@ Here is a summary of the supported (:white_check_mark:) and unsupported (:x:) st
 
 Some HTTPS URLs are automatically recognized as cloud storage:
 
-- **S3**: URLs containing `amazonaws.com` or `r2.cloudflarestorage.com`.
+- **S3**: URLs containing `amazonaws.com`, `r2.cloudflarestorage.com`, or Aliyun OSS S3-compatible endpoints such as `.s3.oss-.aliyuncs.com` and `s3.oss-.aliyuncs.com`.
 - **Azure**: URLs containing `dfs.core.windows.net`, `blob.core.windows.net`, `dfs.fabric.microsoft.com`, or `blob.fabric.microsoft.com`.
 
 These URLs will use the appropriate cloud storage backend instead of the generic HTTP store.
diff --git a/docs/guide/storage/s3.md b/docs/guide/storage/s3.md
index b1761d615..62970ba9d 100644
--- a/docs/guide/storage/s3.md
+++ b/docs/guide/storage/s3.md
@@ -5,7 +5,7 @@ rank: 2
 
 # AWS S3
 
-Sail supports reading and writing data to AWS S3 and S3-compatible object storage services (such as [MinIO](https://min.io/) or Cloudflare R2) using the `s3://`, `s3a://`, or `https://` URI schemes.
+Sail supports reading and writing data to AWS S3 and S3-compatible object storage services such as MinIO, Cloudflare R2, or Aliyun OSS.
 
 For example, `s3://bucket/path/to/data` refers to the `path/to/data` path in the `bucket` bucket.
 Sail determines whether the path refers to a single object or a key prefix.
@@ -163,8 +163,8 @@ df = spark.read.parquet("s3://some-public-bucket/path/to/data")
 You can configure the endpoint and credentials for Cloudflare R2 using environment variables. Here is an example.
 
 ```bash
-export AWS_ACCESS_KEY_ID="smooth"
-export AWS_SECRET_ACCESS_KEY="sailing"
+export AWS_ACCESS_KEY_ID="..."
+export AWS_SECRET_ACCESS_KEY="..."
 export AWS_ENDPOINT="https://my-account-id.r2.cloudflarestorage.com"
 ```
 
@@ -173,11 +173,73 @@ export AWS_ENDPOINT="https://my-account-id.r2.cloudflarestorage.com"
 You can configure the endpoint and credentials for MinIO using environment variables. Here is an example.
 
 ```bash
-export AWS_ACCESS_KEY_ID="smooth"
-export AWS_SECRET_ACCESS_KEY="sailing"
+export AWS_ACCESS_KEY_ID="..."
+export AWS_SECRET_ACCESS_KEY="..."
 export AWS_ENDPOINT="http://localhost:9000"
 ```
 
+### Aliyun OSS
+
+You can configure the endpoint and credentials for Aliyun OSS using environment variables so that you can use the `s3` URL scheme to access Aliyun OSS buckets.
+Here is an example, where `${bucket}` is the bucket name.
+
+```bash
+export AWS_ACCESS_KEY_ID="..."
+export AWS_SECRET_ACCESS_KEY="..."
+export AWS_SESSION_TOKEN="..."  # required if using temporary credentials
+export AWS_REGION="cn-hangzhou"
+export AWS_ENDPOINT="https://${bucket}.s3.oss-cn-hangzhou.aliyuncs.com"
+export AWS_VIRTUAL_HOSTED_STYLE_REQUEST="true"
+```
+
+You can use the `oss` URL scheme to access Aliyun OSS buckets via the S3-compatible endpoint. We will support the `oss` URL scheme via the native Aliyun OSS endpoint in the future.
+You can also use the `https` URL scheme to access Aliyun OSS buckets via the S3-compatible endpoint directly.
+
+
      
+  
    • 
+    oss URL scheme
+    
        
+      
      • 
+        
+      
        • 
+    
      
+  
      • 
+  
    • 
+    https URL scheme
+    
        
+      
      • 
+        
+      
        • 
+      
      • 
+        
+      
        • 
+    
      
+  
      • 
+
    
+
+You can use either long-lived or temporary credentials for Aliyun OSS.
+
+Aliyun OSS S3-compatible endpoints support virtual-hosted-style and path-style URL formats. When you configure `AWS_ENDPOINT`, set `AWS_VIRTUAL_HOSTED_STYLE_REQUEST` to match the endpoint format. The region in the endpoint URL must match the region specified in the `AWS_REGION` environment variable.
+
+The S3-compatible endpoint supports read, write (including multipart upload), list, and delete operations for objects stored in Aliyun OSS.
+
+::: warning
+
+There are a few limitations when using Aliyun OSS S3-compatible access.
+
+- Credential rotation is not supported.
+- The S3-compatible endpoint does not support headers such as `If-None-Match` for object uploads. Some lakehouse formats, such as Delta Lake, may not work because they rely on these headers for atomic operations.
+
+:::
+
+### Other Services
+
 Other storage services that are compatible with the AWS S3 API may be configured similarly. You can refer to the service documentation for more details.
 
 ## Examples
diff --git a/docs/introduction/getting-started/index.md b/docs/introduction/getting-started/index.md
index 0e81f3145..4911716d2 100644
--- a/docs/introduction/getting-started/index.md
+++ b/docs/introduction/getting-started/index.md
@@ -16,19 +16,19 @@ See the [Installation](/introduction/installation/) page for full installation i
 
 ::: code-group
 
-```bash-vue [Spark 4.1 (Client) ]
+```bash-vue [Spark 4.2 (Client) ]
 pip install "pysail=={{ libVersion }}"
-pip install "pyspark-client==4.1.1"
+pip install "pyspark-client==4.2.0"
 ```
 
-```bash-vue [Spark 4.1]
+```bash-vue [Spark 4.2]
 pip install "pysail=={{ libVersion }}"
-pip install "pyspark[connect]==4.1.1"
+pip install "pyspark[connect]==4.2.0"
 ```
 
 ```bash-vue [Spark 3.5]
 pip install "pysail=={{ libVersion }}"
-pip install "pyspark[connect]==3.5.7
+pip install "pyspark[connect]==3.5.9"
 ```
 
 :::
diff --git a/docs/public/favicon.png b/docs/public/favicon.png
index e72bc34ef..061882470 100644
--- a/docs/public/favicon.png
+++ b/docs/public/favicon.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:499c5870c6104aa91ea6f0d42ab3720ac33244f9deb69bf21623d312459ceaf6
-size 1756
+oid sha256:9303a38e99389eb168e137bcf960ef42914c29e8dd4e347708282c576a050c48
+size 3349
diff --git a/docs/reference/changelog/index.md b/docs/reference/changelog/index.md
index dfa869817..cb685a625 100644
--- a/docs/reference/changelog/index.md
+++ b/docs/reference/changelog/index.md
@@ -5,6 +5,177 @@ next: false
 
 # Changelog
 
+## 0.7.0
+
+_August 3, 2026_
+
+- Added support for Spark 4.2 ([#2289](https://github.com/lakehq/sail/pull/2289) and [#2334](https://github.com/lakehq/sail/pull/2334)).
+- Added support for version-aware Spark configuration ([#2296](https://github.com/lakehq/sail/pull/2296)).
+- Added support for `DataFrame.checkpoint()` in the Spark DataFrame API ([#2270](https://github.com/lakehq/sail/pull/2270)).
+- Added support for storage-backed shuffles ([#2232](https://github.com/lakehq/sail/pull/2232), [#2341](https://github.com/lakehq/sail/pull/2341), and [#2347](https://github.com/lakehq/sail/pull/2347)).
+- Added heartbeat support in the Spark Connect server for command execution ([#2323](https://github.com/lakehq/sail/pull/2323)).
+- Improved distributed processing internals ([#2253](https://github.com/lakehq/sail/pull/2253), [#2269](https://github.com/lakehq/sail/pull/2269), [#2329](https://github.com/lakehq/sail/pull/2329), [#2327](https://github.com/lakehq/sail/pull/2327), and [#2326](https://github.com/lakehq/sail/pull/2326)).
+- Added projection pushdown support for explicit repartitioning ([#2183](https://github.com/lakehq/sail/pull/2183)).
+- Improved the performance of queries involving common aggregation expressions ([#2244](https://github.com/lakehq/sail/pull/2244)).
+- Added support for the `DataFrame.inputFiles()` method in the Spark DataFrame API ([#2239](https://github.com/lakehq/sail/pull/2239)).
+- Improved Delta Lake integration ([#2324](https://github.com/lakehq/sail/pull/2324), [#2266](https://github.com/lakehq/sail/pull/2266), [#2263](https://github.com/lakehq/sail/pull/2263), and [#1654](https://github.com/lakehq/sail/pull/1654)).
+- Improved Iceberg integration ([#2292](https://github.com/lakehq/sail/pull/2292), [#2274](https://github.com/lakehq/sail/pull/2274), and [#2229](https://github.com/lakehq/sail/pull/2229)).
+- Added support for bearer token files for the Iceberg REST catalog ([#2288](https://github.com/lakehq/sail/pull/2288)).
+- Improved Hive Metastore integration ([#2230](https://github.com/lakehq/sail/pull/2230)).
+- Added support for explicit AWS Glue catalog IDs ([#2285](https://github.com/lakehq/sail/pull/2285)).
+- Added support for accessing Aliyun OSS S3-compatible endpoints via `oss://` and `https://` URLs ([#2283](https://github.com/lakehq/sail/pull/2283)).
+- Improved datetime format handling for better parity with the Java implementation ([#2099](https://github.com/lakehq/sail/pull/2099) and [#2322](https://github.com/lakehq/sail/pull/2322)).
+- Added support for the `pathGlobFilter` option for all file data sources ([#2284](https://github.com/lakehq/sail/pull/2284)).
+- Added support for the `from_xml` SQL function ([#2098](https://github.com/lakehq/sail/pull/2098)).
+- Improved the following SQL functions ([#2303](https://github.com/lakehq/sail/pull/2303), [#2255](https://github.com/lakehq/sail/pull/2255), [#2260](https://github.com/lakehq/sail/pull/2260), [#2254](https://github.com/lakehq/sail/pull/2254), [#2238](https://github.com/lakehq/sail/pull/2238), [#2190](https://github.com/lakehq/sail/pull/2190), [#2233](https://github.com/lakehq/sail/pull/2233), [#1729](https://github.com/lakehq/sail/pull/1729), [#2157](https://github.com/lakehq/sail/pull/2157), [#2272](https://github.com/lakehq/sail/pull/2272), and [#2312](https://github.com/lakehq/sail/pull/2312)):
+  - `arrays_zip`
+  - `bit_length`
+  - `char_length`
+  - `character_length`
+  - `ceil`
+  - `concat_ws`
+  - `count_min_sketch`
+  - `exists`
+  - `floor`
+  - `forall`
+  - `from_csv`
+  - `hll_sketch_agg`
+  - `len`
+  - `length`
+  - `next_day`
+  - `octet_length`
+  - `regexp_replace`
+  - `schema_of_csv`
+  - `schema_of_json`
+  - `schema_of_variant`
+  - `schema_of_variant_agg`
+  - `theta_sketch_agg`
+  - `to_csv`
+  - `to_xml`
+- Improved ordering semantics for aggregate and window functions ([#2228](https://github.com/lakehq/sail/pull/2228)).
+- Improved the memory catalog by requiring `CASCADE` when dropping non-empty databases ([#2336](https://github.com/lakehq/sail/pull/2336)).
+- Improved OpenAPI client implementations for catalogs ([#2215](https://github.com/lakehq/sail/pull/2215), [#2226](https://github.com/lakehq/sail/pull/2226), and [#2227](https://github.com/lakehq/sail/pull/2227)).
+- Upgraded DataFusion to 54.1.0 ([#2293](https://github.com/lakehq/sail/pull/2293)).
+- Migrated to Rust edition 2024 and increased the MSRV to 1.96.0 ([#2208](https://github.com/lakehq/sail/pull/2208) and [#2224](https://github.com/lakehq/sail/pull/2224)).
+
+### Breaking Changes
+
+- The OpenTelemetry OTLP exporter configuration options were renamed from `telemetry.otlp_*` to `telemetry.exporter.otlp.*` ([#2340](https://github.com/lakehq/sail/pull/2340)).
+- The Parquet `extension` and `fileExtension` data source options were removed. Use `pathGlobFilter` to select files with non-standard suffixes ([#2330](https://github.com/lakehq/sail/pull/2330)).
+
+### Contributors
+
+Huge thanks to [@davidlghellin](https://github.com/davidlghellin), [@rohankumardubey](https://github.com/rohankumardubey) (_first-time contributor_), [@twsl](https://github.com/twsl), [@zemin-piao](https://github.com/zemin-piao), [@anarefolio](https://github.com/anarefolio), [@santosh-d3vpl3x](https://github.com/santosh-d3vpl3x), [@Robin-Everaars](https://github.com/Robin-Everaars) (_first-time contributor_), [@fangchenli](https://github.com/fangchenli) (_first-time contributor_), [@weixiuli](https://github.com/weixiuli) (_first-time contributor_), [@nis12ram](https://github.com/nis12ram), and [@ShiroKSH](https://github.com/ShiroKSH) (_first-time contributor_) for your contributions!
+
+## 0.6.6
+
+_July 7, 2026_
+
+- Improved distributed query execution and fixed a few issues around query plans in cluster mode ([#2171](https://github.com/lakehq/sail/pull/2171), [#2182](https://github.com/lakehq/sail/pull/2182), [#2191](https://github.com/lakehq/sail/pull/2191), [#2192](https://github.com/lakehq/sail/pull/2192), [#2194](https://github.com/lakehq/sail/pull/2194), [#2199](https://github.com/lakehq/sail/pull/2199), [#2200](https://github.com/lakehq/sail/pull/2200), [#2201](https://github.com/lakehq/sail/pull/2201), and [#2207](https://github.com/lakehq/sail/pull/2207)).
+- Improved Delta Lake integration ([#2195](https://github.com/lakehq/sail/pull/2195)).
+- Added support for the `SHOW FUNCTIONS` and `DESCRIBE FUNCTION` SQL statements ([#2151](https://github.com/lakehq/sail/pull/2151)).
+- Added support for recovering Python data source options from table properties ([#2203](https://github.com/lakehq/sail/pull/2203)).
+- Improved the `PIVOT` clause in SQL queries ([#2197](https://github.com/lakehq/sail/pull/2197)).
+- Improved the following SQL functions ([#2158](https://github.com/lakehq/sail/pull/2158), [#2174](https://github.com/lakehq/sail/pull/2174), [#2175](https://github.com/lakehq/sail/pull/2175), [#2180](https://github.com/lakehq/sail/pull/2180), [#2187](https://github.com/lakehq/sail/pull/2187), [#2188](https://github.com/lakehq/sail/pull/2188), [#2189](https://github.com/lakehq/sail/pull/2189), [#2196](https://github.com/lakehq/sail/pull/2196), [#2197](https://github.com/lakehq/sail/pull/2197), and [#2204](https://github.com/lakehq/sail/pull/2204)):
+  - `base64`
+  - `bin`
+  - `bround`
+  - `collect_list`
+  - `collect_set`
+  - `concat`
+  - `concat_ws`
+  - `decode`
+  - `encode`
+  - `from_json`
+  - `make_timestamp_ntz`
+  - `make_ym_interval`
+  - `schema_of_variant`
+  - `struct`
+  - `time_diff`
+  - `to_csv`
+  - `to_json`
+  - `unbase64`
+
+### Contributors
+
+Huge thanks to [@davidlghellin](https://github.com/davidlghellin) for your contributions!
+
+## 0.6.5
+
+_June 29, 2026_
+
+- Improved Delta Lake integration ([#1957](https://github.com/lakehq/sail/pull/1957), [#2097](https://github.com/lakehq/sail/pull/2097), [#2078](https://github.com/lakehq/sail/pull/2078), [#2110](https://github.com/lakehq/sail/pull/2110), [#2122](https://github.com/lakehq/sail/pull/2122), and [#2148](https://github.com/lakehq/sail/pull/2148)).
+- Improved Iceberg integration ([#1929](https://github.com/lakehq/sail/pull/1929), [#2097](https://github.com/lakehq/sail/pull/2097), [#2078](https://github.com/lakehq/sail/pull/2078), [#2119](https://github.com/lakehq/sail/pull/2119), and [#2134](https://github.com/lakehq/sail/pull/2134)).
+- Improved the OneLake catalog by supporting its Unity Catalog and Iceberg REST endpoints and supporting more authentication methods ([#2141](https://github.com/lakehq/sail/pull/2141)).
+- Fixed an issue with column default metadata handling for the Hive Metastore catalog ([#2058](https://github.com/lakehq/sail/pull/2058)).
+- Improved the internals for data sources ([#2047](https://github.com/lakehq/sail/pull/2047), [#2050](https://github.com/lakehq/sail/pull/2050), [#2069](https://github.com/lakehq/sail/pull/2069), [#2070](https://github.com/lakehq/sail/pull/2070), and [#2077](https://github.com/lakehq/sail/pull/2077)).
+- Added support for various write modes for file-based data sources ([#2053](https://github.com/lakehq/sail/pull/2053) and [#2068](https://github.com/lakehq/sail/pull/2068)).
+- Added support for the `USING` clause to create writable temporary views for data sources ([#2059](https://github.com/lakehq/sail/pull/2059)).
+- Added support for Parquet content-defined chunking options ([#2125](https://github.com/lakehq/sail/pull/2125)).
+- Added support for the `PIVOT` clause in SQL and the `pivot()` method in the Spark DataFrame API ([#2120](https://github.com/lakehq/sail/pull/2120) and [#2138](https://github.com/lakehq/sail/pull/2138)).
+- Added support for named window clauses in SQL ([#2039](https://github.com/lakehq/sail/pull/2039)).
+- Added support for the following SQL functions ([#2032](https://github.com/lakehq/sail/pull/2032), [#2006](https://github.com/lakehq/sail/pull/2006), [#2063](https://github.com/lakehq/sail/pull/2063), [#2094](https://github.com/lakehq/sail/pull/2094), [#2054](https://github.com/lakehq/sail/pull/2054), [#2057](https://github.com/lakehq/sail/pull/2057), [#2103](https://github.com/lakehq/sail/pull/2103), [#2106](https://github.com/lakehq/sail/pull/2106), [#2109](https://github.com/lakehq/sail/pull/2109), [#2123](https://github.com/lakehq/sail/pull/2123), and [#2127](https://github.com/lakehq/sail/pull/2127)):
+  - `aggregate`
+  - `exists`
+  - `filter`
+  - `forall`
+  - `transform`
+  - `to_char`
+  - `to_xml`
+  - `to_varchar`
+  - `try_to_time`
+  - `window`
+  - `window_time`
+- Improved the following SQL functions ([#1891](https://github.com/lakehq/sail/pull/1891), [#1962](https://github.com/lakehq/sail/pull/1962), [#2019](https://github.com/lakehq/sail/pull/2019), [#2028](https://github.com/lakehq/sail/pull/2028), [#2030](https://github.com/lakehq/sail/pull/2030), [#2031](https://github.com/lakehq/sail/pull/2031), [#2040](https://github.com/lakehq/sail/pull/2040), [#2054](https://github.com/lakehq/sail/pull/2054), [#2074](https://github.com/lakehq/sail/pull/2074), [#2082](https://github.com/lakehq/sail/pull/2082), [#2086](https://github.com/lakehq/sail/pull/2086), [#2094](https://github.com/lakehq/sail/pull/2094), [#2095](https://github.com/lakehq/sail/pull/2095), [#2100](https://github.com/lakehq/sail/pull/2100), [#2104](https://github.com/lakehq/sail/pull/2104), [#2121](https://github.com/lakehq/sail/pull/2121), [#2130](https://github.com/lakehq/sail/pull/2130), [#2142](https://github.com/lakehq/sail/pull/2142), and [#1225](https://github.com/lakehq/sail/pull/1225)):
+  - `array_position`
+  - `array_sort`
+  - `bin`
+  - `concat`
+  - `concat_ws`
+  - `get_json_object`
+  - `hll_sketch_agg`
+  - `hll_union`
+  - `hll_union_agg`
+  - `inline`
+  - `inline_outer`
+  - `kurtosis`
+  - `map_entries`
+  - `mode`
+  - `negative`
+  - `percentile_disc`
+  - `regr_avgx`
+  - `regr_avgy`
+  - `regr_count`
+  - `regr_intercept`
+  - `regr_r2`
+  - `regr_slope`
+  - `regr_sxx`
+  - `regr_sxy`
+  - `regr_syy`
+  - `regexp_extract`
+  - `schema_of_json`
+  - `skewness`
+  - `timestampadd`
+  - `theta_sketch_agg`
+  - `theta_union`
+  - `theta_union_agg`
+  - `to_number`
+  - `to_time`
+  - `to_timestamp`
+  - `try_to_timestamp`
+  - `xxhash64`
+- Improved variant data type handling ([#2142](https://github.com/lakehq/sail/pull/2142)).
+- Improved column handling in the Spark DataFrame API ([#2142](https://github.com/lakehq/sail/pull/2142)).
+- Added support for the `?` parameter marker in SQL queries ([#2142](https://github.com/lakehq/sail/pull/2142)).
+- Improved Spark compatibility for `EXPLAIN` output ([#2114](https://github.com/lakehq/sail/pull/2114)).
+- Fixed an issue with case-insensitive function name matching ([#2131](https://github.com/lakehq/sail/pull/2131)).
+- Fixed an issue where pending tasks could fail while workers were still launching ([#2081](https://github.com/lakehq/sail/pull/2081)).
+- Upgraded DataFusion to 54.0.0 and Arrow to 58.3.0 ([#1956](https://github.com/lakehq/sail/pull/1956), [#2065](https://github.com/lakehq/sail/pull/2065), and [#2066](https://github.com/lakehq/sail/pull/2066)).
+
+### Contributors
+
+Huge thanks to [@davidlghellin](https://github.com/davidlghellin), [@anarefolio](https://github.com/anarefolio), [@twsl](https://github.com/twsl), [@nis12ram](https://github.com/nis12ram), [@zemin-piao](https://github.com/zemin-piao), [@radmirnovii](https://github.com/radmirnovii) (_first-time contributor_), [@benzsevern](https://github.com/benzsevern) (_first-time contributor_), and [@ctydw123](https://github.com/ctydw123) (_first-time contributor_) for your contributions!
+
 ## 0.6.4
 
 _June 6, 2026_
@@ -46,7 +217,7 @@ _June 6, 2026_
   - `pmod`
   - `to_date`
 - Improved datetime exception handling ([#1970](https://github.com/lakehq/sail/pull/1970)).
-- Improved the performance for datetime formatting string parsing ([#1925](https://github.com/lakehq/sail/pull/1925)).
+- Improved the performance of datetime formatting string parsing ([#1925](https://github.com/lakehq/sail/pull/1925)).
 - Improved complex type and nullability handling logic ([#1950](https://github.com/lakehq/sail/pull/1950)).
 - Added support for referring to nested struct fields within arrays ([#2029](https://github.com/lakehq/sail/pull/2029)).
 - Fixed an issue with rendering the `TIME` data type as strings ([#1978](https://github.com/lakehq/sail/pull/1978)).
@@ -273,7 +444,7 @@ _March 2, 2026_
 - Added support for the JDBC data source ([#1379](https://github.com/lakehq/sail/pull/1379), [#1389](https://github.com/lakehq/sail/pull/1389), and [#1446](https://github.com/lakehq/sail/pull/1446)).
 - Added support for the `SELECT * FROM .` SQL query syntax ([#1439](https://github.com/lakehq/sail/pull/1439)).
 - Added support for the `IDENTIFIER` clause in SQL queries ([#1441](https://github.com/lakehq/sail/pull/1441)).
-- Added support for passing DataFrame as arguments to SQL queries in PySpark ([#1426](https://github.com/lakehq/sail/pull/1426)).
+- Added support for passing DataFrames as arguments to SQL queries in PySpark ([#1426](https://github.com/lakehq/sail/pull/1426)).
 - Improved Delta Lake integration ([#1352](https://github.com/lakehq/sail/pull/1352), [#1359](https://github.com/lakehq/sail/pull/1359), and [#1415](https://github.com/lakehq/sail/pull/1415)).
 - Improved the join reorder optimizer ([#1248](https://github.com/lakehq/sail/pull/1248)).
 - Added support for the following SQL functions ([#1358](https://github.com/lakehq/sail/pull/1358) and [#1398](https://github.com/lakehq/sail/pull/1398)):
@@ -304,7 +475,7 @@ _March 2, 2026_
 
 ### Contributors
 
-Huge thanks to [@tamirkifle](https://github.com/tamirkifle)(_first-time contributor_), [@davidlghellin](https://github.com/davidlghellin), [@santosh-d3vpl3x](https://github.com/santosh-d3vpl3x), [@zemin-piao](https://github.com/zemin-piao), [@james-willis](https://github.com/james-willis), and [@pomykalakyle](https://github.com/pomykalakyle) for your contributions!
+Huge thanks to [@tamirkifle](https://github.com/tamirkifle) (_first-time contributor_), [@davidlghellin](https://github.com/davidlghellin), [@santosh-d3vpl3x](https://github.com/santosh-d3vpl3x), [@zemin-piao](https://github.com/zemin-piao), [@james-willis](https://github.com/james-willis), and [@pomykalakyle](https://github.com/pomykalakyle) for your contributions!
 
 ## 0.5.1
 
@@ -353,15 +524,15 @@ Huge thanks to [@davidlghellin](https://github.com/davidlghellin), [@pomykalakyl
 _February 6, 2026_
 
 - Redesigned the control plane for distributed query execution ([#1164](https://github.com/lakehq/sail/pull/1164), [#1242](https://github.com/lakehq/sail/pull/1242), [#1247](https://github.com/lakehq/sail/pull/1247), [#1265](https://github.com/lakehq/sail/pull/1265), and [#1280](https://github.com/lakehq/sail/pull/1280)).
-- Added support for system catalog ([#1216](https://github.com/lakehq/sail/pull/1216)).
-- Added support for AWS Glue catalog ([#1254](https://github.com/lakehq/sail/pull/1254) and [#1279](https://github.com/lakehq/sail/pull/1279)).
-- Added support for OneLake catalog ([#1217](https://github.com/lakehq/sail/pull/1217) and [#1228](https://github.com/lakehq/sail/pull/1228)).
-- Added support for partition transforms for Iceberg REST catalog ([#1269](https://github.com/lakehq/sail/pull/1269)).
+- Added support for the system catalog ([#1216](https://github.com/lakehq/sail/pull/1216)).
+- Added support for the AWS Glue catalog ([#1254](https://github.com/lakehq/sail/pull/1254) and [#1279](https://github.com/lakehq/sail/pull/1279)).
+- Added support for the OneLake catalog ([#1217](https://github.com/lakehq/sail/pull/1217) and [#1228](https://github.com/lakehq/sail/pull/1228)).
+- Added support for partition transforms for the Iceberg REST catalog ([#1269](https://github.com/lakehq/sail/pull/1269)).
 - Added support for the `SHOW CATALOGS` and `USE CATALOG` SQL statements ([#1288](https://github.com/lakehq/sail/pull/1288)).
 - Improved Delta Lake integration ([#1222](https://github.com/lakehq/sail/pull/1222)).
 - Improved Iceberg integration ([#1169](https://github.com/lakehq/sail/pull/1169)).
 - Added support for the `CREATE TABLE ... AS SELECT ...` (CTAS) statement ([#1236](https://github.com/lakehq/sail/pull/1236)).
-- Added support for the `inferSchema` option for CSV data source ([#1223](https://github.com/lakehq/sail/pull/1223)).
+- Added support for the `inferSchema` option for the CSV data source ([#1223](https://github.com/lakehq/sail/pull/1223)).
 - Added support for `DataFrame.colRegex()` in the Spark DataFrame API ([#1243](https://github.com/lakehq/sail/pull/1243)).
 - Added support for `StructType.toDDL()` in the Spark DataFrame API ([#1285](https://github.com/lakehq/sail/pull/1285)).
 - Added support for the following SQL functions ([#1200](https://github.com/lakehq/sail/pull/1200), [#1206](https://github.com/lakehq/sail/pull/1206), [#1218](https://github.com/lakehq/sail/pull/1218), [#1253](https://github.com/lakehq/sail/pull/1253), [#1258](https://github.com/lakehq/sail/pull/1258), [#1263](https://github.com/lakehq/sail/pull/1263), [#1268](https://github.com/lakehq/sail/pull/1268), and [#1276](https://github.com/lakehq/sail/pull/1276)):
@@ -391,7 +562,7 @@ _February 6, 2026_
   - `flatten`
 - Added support for negation and the `signum` SQL function for interval data types ([#1275](https://github.com/lakehq/sail/pull/1275)).
 - Fixed issues with the `Column.try_cast()` method in the Spark DataFrame API to handle invalid date and timestamp values correctly ([#1221](https://github.com/lakehq/sail/pull/1221)).
-- Improved the `DataFrame.randomSplit()` method in the Spark DataFrame API to ensured deterministic order ([#1235](https://github.com/lakehq/sail/pull/1235)).
+- Improved the `DataFrame.randomSplit()` method in the Spark DataFrame API to ensure deterministic order ([#1235](https://github.com/lakehq/sail/pull/1235)).
 - Improved the join reorder optimizer ([#1234](https://github.com/lakehq/sail/pull/1234)).
 - Added memory and disk configuration options ([#1311](https://github.com/lakehq/sail/pull/1311)).
 - Added support for inferring default catalog when only one catalog is configured ([#1311](https://github.com/lakehq/sail/pull/1311)).
@@ -414,10 +585,10 @@ _January 13, 2026_
 - Improved Delta Lake integration ([#1146](https://github.com/lakehq/sail/pull/1146), [#1159](https://github.com/lakehq/sail/pull/1159), [#1158](https://github.com/lakehq/sail/pull/1158), and [#1161](https://github.com/lakehq/sail/pull/1161)).
 - Improved the internals for session management ([#1138](https://github.com/lakehq/sail/pull/1138)).
 - Added support for reading CSV files with truncated rows ([#1185](https://github.com/lakehq/sail/pull/1185)).
-- Added the configuration option for default parallelism ([#1198](https://github.com/lakehq/sail/pull/1198)).
+- Added a configuration option for default parallelism ([#1198](https://github.com/lakehq/sail/pull/1198)).
 - Added the `percentile_cont` SQL aggregate function ([#1188](https://github.com/lakehq/sail/pull/1188)).
 - Added support for non-literal expressions for map extraction ([#1193](https://github.com/lakehq/sail/pull/1193)).
-- Added support for wildcard for the `struct` SQL function ([#1197](https://github.com/lakehq/sail/pull/1197)).
+- Added support for wildcards in the `struct` SQL function ([#1197](https://github.com/lakehq/sail/pull/1197)).
 - Fixed an issue with null value handling in the `map_concat` SQL function ([#1194](https://github.com/lakehq/sail/pull/1194)).
 - Updated the PySpark compatibility checker example and function support status ([#1127](https://github.com/lakehq/sail/pull/1127)).
 - Updated the TPC-H benchmark example ([#1179](https://github.com/lakehq/sail/pull/1179)).
@@ -465,7 +636,7 @@ _December 12, 2025_
 - Improved the output of the `EXPLAIN` statement ([#1110](https://github.com/lakehq/sail/pull/1110)).
 - Fixed a few shuffle planning issues in distributed query execution ([#1111](https://github.com/lakehq/sail/pull/1111)).
 - Fixed an issue with the `LIMIT` clause in distributed query execution ([#1121](https://github.com/lakehq/sail/pull/1121)).
-- Improved data source implementation ([#1099](https://github.com/lakehq/sail/pull/1099)).
+- Improved the data source implementation ([#1099](https://github.com/lakehq/sail/pull/1099)).
 
 ### Contributors
 
@@ -534,7 +705,7 @@ Huge thanks to [@davidlghellin](https://github.com/davidlghellin) for the contri
 _October 29, 2025_
 
 - Added basic support for reading and writing Iceberg tables ([#944](https://github.com/lakehq/sail/pull/944), [#987](https://github.com/lakehq/sail/pull/987), [#976](https://github.com/lakehq/sail/pull/976), [#994](https://github.com/lakehq/sail/pull/994), and [#997](https://github.com/lakehq/sail/pull/997)).
-- Added support for Iceberg REST catalog ([#961](https://github.com/lakehq/sail/pull/961), [#974](https://github.com/lakehq/sail/pull/974), [#993](https://github.com/lakehq/sail/pull/993), and [#995](https://github.com/lakehq/sail/pull/995)).
+- Added support for the Iceberg REST catalog ([#961](https://github.com/lakehq/sail/pull/961), [#974](https://github.com/lakehq/sail/pull/974), [#993](https://github.com/lakehq/sail/pull/993), and [#995](https://github.com/lakehq/sail/pull/995)).
 - Improved Delta Lake integration ([#921](https://github.com/lakehq/sail/pull/921)).
 - Added support for multiple arguments for the `count_distinct` SQL function ([#957](https://github.com/lakehq/sail/pull/957)).
 - Added guide for HDFS Kerberos authentication ([#992](https://github.com/lakehq/sail/pull/992)).
@@ -784,7 +955,7 @@ _August 8, 2025_
 
 ### Contributors
 
-Shoutout to [@SparkApplicationMaster](https://github.com/SparkApplicationMaster) for contributions across bug fixes, features, and enhancements! Huge thanks to [@rafafrdz](https://github.com/rafafrdz), [@davidlghellin](https://github.com/davidlghellin), [@anhvdq](https://github.com/anhvdq) (_first-time contributor_), and [@jamesfricker](https://github.com/jamesfricker) (_first-time contributor_), for helping to further extend our parity with Spark SQL functions!
+Shoutout to [@SparkApplicationMaster](https://github.com/SparkApplicationMaster) for contributions across bug fixes, features, and enhancements! Huge thanks to [@rafafrdz](https://github.com/rafafrdz), [@davidlghellin](https://github.com/davidlghellin), [@anhvdq](https://github.com/anhvdq) (_first-time contributor_), and [@jamesfricker](https://github.com/jamesfricker) (_first-time contributor_) for helping to further extend our parity with Spark SQL functions!
 
 ## 0.3.1
 
@@ -926,8 +1097,8 @@ We continued extending coverage for Spark SQL functions and the Spark DataFrame
   - `base64`
   - `unbase64`
   - `weekofyear`
-- Added support for `mapInPandas()` and `mapInArrow()` for Spark DataFrame ([#310](https://github.com/lakehq/sail/pull/310)).
-- Added support for `applyInPandas()` for grouped and co-grouped Spark DataFrame ([#313](https://github.com/lakehq/sail/pull/313)).
+- Added support for `mapInPandas()` and `mapInArrow()` for Spark DataFrames ([#310](https://github.com/lakehq/sail/pull/310)).
+- Added support for `applyInPandas()` for grouped and co-grouped Spark DataFrames ([#313](https://github.com/lakehq/sail/pull/313)).
 
 ### Breaking Changes
 
@@ -1005,7 +1176,7 @@ _October 03, 2024_
   - `sum_distinct`
 - Added support for HDFS ([#196](https://github.com/lakehq/sail/pull/196)).
 - Added support for parsing value prefixes followed by whitespace ([#218](https://github.com/lakehq/sail/pull/218) and [lakehq/sqlparser-rs#6](https://github.com/lakehq/sqlparser-rs/pull/6)).
-- Added basic support for Python UDAF ([#214](https://github.com/lakehq/sail/pull/214)).
+- Added basic support for Python UDAFs ([#214](https://github.com/lakehq/sail/pull/214)).
 
 ### Contributors
 
diff --git a/package.json b/package.json
index 21ed918c6..8f1cdda94 100644
--- a/package.json
+++ b/package.json
@@ -11,37 +11,37 @@
   "devDependencies": {
     "@eslint/js": "^10.0.1",
     "@ianvs/prettier-plugin-sort-imports": "^4.7.1",
-    "@tailwindcss/vite": "^4.3.1",
+    "@tailwindcss/vite": "^4.3.3",
     "@types/js-yaml": "^4.0.9",
     "@types/markdown-it": "^14.1.2",
     "@types/node": "^24.12.2",
-    "@typescript-eslint/eslint-plugin": "^8.61.1",
-    "@typescript-eslint/parser": "^8.61.1",
+    "@typescript-eslint/eslint-plugin": "^8.65.0",
+    "@typescript-eslint/parser": "^8.65.0",
     "@viz-js/viz": "^3.28.0",
-    "autoprefixer": "^10.5.0",
-    "eslint": "^10.5.0",
+    "autoprefixer": "^10.5.4",
+    "eslint": "^10.8.0",
     "eslint-config-prettier": "^10.1.8",
-    "eslint-plugin-vue": "^10.9.2",
+    "eslint-plugin-vue": "^10.10.0",
     "fast-glob": "^3.3.3",
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.2.0",
-    "markdown-it": "^14.2.0",
+    "markdown-it": "^14.3.0",
     "markdown-it-deflist": "^3.0.1",
     "markdown-it-footnote": "^4.0.0",
-    "prettier": "^3.8.4",
-    "prettier-plugin-tailwindcss": "^0.8.0",
+    "prettier": "^3.9.6",
+    "prettier-plugin-tailwindcss": "^0.8.1",
     "prettier-plugin-vitepress": "^0.0.10",
-    "sql-formatter": "^15.8.1",
-    "svgo": "^4.0.1",
-    "tailwindcss": "^4.3.1",
+    "sql-formatter": "^15.8.2",
+    "svgo": "^4.0.2",
+    "tailwindcss": "^4.3.3",
     "typescript": "^6.0.3",
-    "typescript-eslint": "^8.61.1",
-    "vega": "^6.2.0",
+    "typescript-eslint": "^8.65.0",
+    "vega": "^6.3.1",
     "vite": "^5.4.21",
     "vitepress": "^1.6.4",
-    "vue": "^3.5.38",
+    "vue": "^3.5.40",
     "vue-eslint-parser": "^10.4.1",
-    "vue-tsc": "^3.3.5"
+    "vue-tsc": "^3.3.9"
   },
   "prettier": {
     "plugins": [
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 42cbf0cbf..b260e11aa 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -10,13 +10,13 @@ importers:
     devDependencies:
       '@eslint/js':
         specifier: ^10.0.1
-        version: 10.0.1(eslint@10.5.0(jiti@2.7.0))
+        version: 10.0.1(eslint@10.8.0(jiti@2.7.0))
       '@ianvs/prettier-plugin-sort-imports':
         specifier: ^4.7.1
-        version: 4.7.1(@vue/compiler-sfc@3.5.38)(prettier@3.8.4)
+        version: 4.7.1(@vue/compiler-sfc@3.5.40)(prettier@3.9.6)
       '@tailwindcss/vite':
-        specifier: ^4.3.1
-        version: 4.3.1(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))
+        specifier: ^4.3.3
+        version: 4.3.3(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))
       '@types/js-yaml':
         specifier: ^4.0.9
         version: 4.0.9
@@ -27,26 +27,26 @@ importers:
         specifier: ^24.12.2
         version: 24.12.2
       '@typescript-eslint/eslint-plugin':
-        specifier: ^8.61.1
-        version: 8.61.1(@typescript-eslint/parser@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
+        specifier: ^8.65.0
+        version: 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
       '@typescript-eslint/parser':
-        specifier: ^8.61.1
-        version: 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
+        specifier: ^8.65.0
+        version: 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
       '@viz-js/viz':
         specifier: ^3.28.0
         version: 3.28.0
       autoprefixer:
-        specifier: ^10.5.0
-        version: 10.5.0(postcss@8.5.15)
+        specifier: ^10.5.4
+        version: 10.5.4(postcss@8.5.22)
       eslint:
-        specifier: ^10.5.0
-        version: 10.5.0(jiti@2.7.0)
+        specifier: ^10.8.0
+        version: 10.8.0(jiti@2.7.0)
       eslint-config-prettier:
         specifier: ^10.1.8
-        version: 10.1.8(eslint@10.5.0(jiti@2.7.0))
+        version: 10.1.8(eslint@10.8.0(jiti@2.7.0))
       eslint-plugin-vue:
-        specifier: ^10.9.2
-        version: 10.9.2(@typescript-eslint/parser@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.5.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.5.0(jiti@2.7.0)))
+        specifier: ^10.10.0
+        version: 10.10.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)))
       fast-glob:
         specifier: ^3.3.3
         version: 3.3.3
@@ -57,8 +57,8 @@ importers:
         specifier: ^4.2.0
         version: 4.2.0
       markdown-it:
-        specifier: ^14.2.0
-        version: 14.2.0
+        specifier: ^14.3.0
+        version: 14.3.0
       markdown-it-deflist:
         specifier: ^3.0.1
         version: 3.0.1
@@ -66,47 +66,47 @@ importers:
         specifier: ^4.0.0
         version: 4.0.0
       prettier:
-        specifier: ^3.8.4
-        version: 3.8.4
+        specifier: ^3.9.6
+        version: 3.9.6
       prettier-plugin-tailwindcss:
-        specifier: ^0.8.0
-        version: 0.8.0(@ianvs/prettier-plugin-sort-imports@4.7.1(@vue/compiler-sfc@3.5.38)(prettier@3.8.4))(prettier@3.8.4)
+        specifier: ^0.8.1
+        version: 0.8.1(@ianvs/prettier-plugin-sort-imports@4.7.1(@vue/compiler-sfc@3.5.40)(prettier@3.9.6))(prettier@3.9.6)
       prettier-plugin-vitepress:
         specifier: ^0.0.10
-        version: 0.0.10(prettier@3.8.4)
+        version: 0.0.10(prettier@3.9.6)
       sql-formatter:
-        specifier: ^15.8.1
-        version: 15.8.1
+        specifier: ^15.8.2
+        version: 15.8.2
       svgo:
-        specifier: ^4.0.1
-        version: 4.0.1
+        specifier: ^4.0.2
+        version: 4.0.2
       tailwindcss:
-        specifier: ^4.3.1
-        version: 4.3.1
+        specifier: ^4.3.3
+        version: 4.3.3
       typescript:
         specifier: ^6.0.3
         version: 6.0.3
       typescript-eslint:
-        specifier: ^8.61.1
-        version: 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
+        specifier: ^8.65.0
+        version: 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
       vega:
-        specifier: ^6.2.0
-        version: 6.2.0
+        specifier: ^6.3.1
+        version: 6.3.1
       vite:
         specifier: ^5.4.21
         version: 5.4.21(@types/node@24.12.2)(lightningcss@1.32.0)
       vitepress:
         specifier: ^1.6.4
-        version: 1.6.4(@algolia/client-search@5.50.2)(@types/node@24.12.2)(lightningcss@1.32.0)(postcss@8.5.15)(search-insights@2.17.3)(typescript@6.0.3)
+        version: 1.6.4(@algolia/client-search@5.50.2)(@types/node@24.12.2)(lightningcss@1.32.0)(postcss@8.5.22)(search-insights@2.17.3)(typescript@6.0.3)
       vue:
-        specifier: ^3.5.38
-        version: 3.5.38(typescript@6.0.3)
+        specifier: ^3.5.40
+        version: 3.5.40(typescript@6.0.3)
       vue-eslint-parser:
         specifier: ^10.4.1
-        version: 10.4.1(eslint@10.5.0(jiti@2.7.0))
+        version: 10.4.1(eslint@10.8.0(jiti@2.7.0))
       vue-tsc:
-        specifier: ^3.3.5
-        version: 3.3.5(typescript@6.0.3)
+        specifier: ^3.3.9
+        version: 3.3.9(typescript@6.0.3)
 
 packages:
 
@@ -224,6 +224,11 @@ packages:
     engines: {node: '>=6.0.0'}
     hasBin: true
 
+  '@babel/parser@7.29.8':
+    resolution: {integrity: sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==}
+    engines: {node: '>=6.0.0'}
+    hasBin: true
+
   '@babel/template@7.28.6':
     resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==}
     engines: {node: '>=6.9.0'}
@@ -240,6 +245,10 @@ packages:
     resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==}
     engines: {node: '>=6.9.0'}
 
+  '@babel/types@7.29.8':
+    resolution: {integrity: sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==}
+    engines: {node: '>=6.9.0'}
+
   '@docsearch/css@3.8.2':
     resolution: {integrity: sha512-y05ayQFyUmCXze79+56v/4HpycYF3uFqB78pLPrSV5ZKAlDuIAAJNhaRi8tTdRNXh05yxX/TyNnzD6LwSM89vQ==}
 
@@ -401,8 +410,8 @@ packages:
     cpu: [x64]
     os: [win32]
 
-  '@eslint-community/eslint-utils@4.9.1':
-    resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==}
+  '@eslint-community/eslint-utils@4.10.1':
+    resolution: {integrity: sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     peerDependencies:
       eslint: ^6.0.0 || ^7.0.0 || >=8.0.0
@@ -415,8 +424,8 @@ packages:
     resolution: {integrity: sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==}
     engines: {node: ^20.19.0 || ^22.13.0 || >=24}
 
-  '@eslint/config-helpers@0.6.0':
-    resolution: {integrity: sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA==}
+  '@eslint/config-helpers@0.7.0':
+    resolution: {integrity: sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==}
     engines: {node: ^20.19.0 || ^22.13.0 || >=24}
 
   '@eslint/core@1.2.1':
@@ -661,65 +670,65 @@ packages:
   '@shikijs/vscode-textmate@10.0.2':
     resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==}
 
-  '@tailwindcss/node@4.3.1':
-    resolution: {integrity: sha512-6NDaqRoAMSXD1mr/RXu0HBvNE9a2n5tHPsxu9XHLws8o4Twes5rBM2205SUUiJ9goAtadrN6xTGX0UDEwp/N4A==}
+  '@tailwindcss/node@4.3.3':
+    resolution: {integrity: sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==}
 
-  '@tailwindcss/oxide-android-arm64@4.3.1':
-    resolution: {integrity: sha512-SVlyf61g374l5cHyg8x9kf5xmLcOaxvOTsbsqDnSsDJaKOEFZ7GCvi84VAVGpxojYOs1+3K6M0UjXfqPU8vmOQ==}
+  '@tailwindcss/oxide-android-arm64@4.3.3':
+    resolution: {integrity: sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==}
     engines: {node: '>= 20'}
     cpu: [arm64]
     os: [android]
 
-  '@tailwindcss/oxide-darwin-arm64@4.3.1':
-    resolution: {integrity: sha512-hVnWLwv+e/l7c4WKyVtHVrIPvYdqWHjRB3MDIqARynzFtnQg85kmQEFCbV9Ja0VVx4xXTIiDWY60Y7iz/iNoDA==}
+  '@tailwindcss/oxide-darwin-arm64@4.3.3':
+    resolution: {integrity: sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==}
     engines: {node: '>= 20'}
     cpu: [arm64]
     os: [darwin]
 
-  '@tailwindcss/oxide-darwin-x64@4.3.1':
-    resolution: {integrity: sha512-Cf7abu0WVgbhU7ANgPUnSAvm7nCvMweusHb8FnaHlLfv/Caq4GYaEZg7ZImzzmjx4lIAfuS8q+eLIS7A7IzxIg==}
+  '@tailwindcss/oxide-darwin-x64@4.3.3':
+    resolution: {integrity: sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==}
     engines: {node: '>= 20'}
     cpu: [x64]
     os: [darwin]
 
-  '@tailwindcss/oxide-freebsd-x64@4.3.1':
-    resolution: {integrity: sha512-ZZqzX2Y+GXtXXfqSfpJhDm60OoZfvLHLCgm+J7NVqgHHJjG/m9ugZI77RwTsVd4fnBJuCFP6Ae6kTJb71UdS8g==}
+  '@tailwindcss/oxide-freebsd-x64@4.3.3':
+    resolution: {integrity: sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==}
     engines: {node: '>= 20'}
     cpu: [x64]
     os: [freebsd]
 
-  '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.1':
-    resolution: {integrity: sha512-/Ah/xik0LaMYfv9DZ0S/t4pBlBNYOcqtRwusjgovHkvT8ixueWCLyJjsaF5kQIckjb4IT8Q6K6p/iPmZMixYgg==}
+  '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3':
+    resolution: {integrity: sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==}
     engines: {node: '>= 20'}
     cpu: [arm]
     os: [linux]
 
-  '@tailwindcss/oxide-linux-arm64-gnu@4.3.1':
-    resolution: {integrity: sha512-gqdFoVJlw444GvpnheZLHmvTzSxI/cOUUh2KSNejQjTcYkW062SVD+En0rUgD+QV91bz1XGIGtt1HJd48xUGbQ==}
+  '@tailwindcss/oxide-linux-arm64-gnu@4.3.3':
+    resolution: {integrity: sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==}
     engines: {node: '>= 20'}
     cpu: [arm64]
     os: [linux]
 
-  '@tailwindcss/oxide-linux-arm64-musl@4.3.1':
-    resolution: {integrity: sha512-Bwv9KwOvE0VKa86xPFif9b9c3Y1NxOV1P0gLti/IYaWEsQYZXDlxfGEtA8mdDZ7SG3wyNXAWYT5SIn3giL57oA==}
+  '@tailwindcss/oxide-linux-arm64-musl@4.3.3':
+    resolution: {integrity: sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==}
     engines: {node: '>= 20'}
     cpu: [arm64]
     os: [linux]
 
-  '@tailwindcss/oxide-linux-x64-gnu@4.3.1':
-    resolution: {integrity: sha512-Ymi8O8T15HYQdOUWUtTI6ldN0neHP85FC+Qz32xTcZ7iJXtem/x8ITev0o1e9e5rkqj4lONZfTRLvkmin1+tKg==}
+  '@tailwindcss/oxide-linux-x64-gnu@4.3.3':
+    resolution: {integrity: sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==}
     engines: {node: '>= 20'}
     cpu: [x64]
     os: [linux]
 
-  '@tailwindcss/oxide-linux-x64-musl@4.3.1':
-    resolution: {integrity: sha512-M+P/91qJ6uILLw4k2G93GMDRAXj61SMvFQYt39AqvUqYgExXpLL5aepfns7sj4HiAQeolirQF9E0lzRvdf4zPQ==}
+  '@tailwindcss/oxide-linux-x64-musl@4.3.3':
+    resolution: {integrity: sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==}
     engines: {node: '>= 20'}
     cpu: [x64]
     os: [linux]
 
-  '@tailwindcss/oxide-wasm32-wasi@4.3.1':
-    resolution: {integrity: sha512-zsM8uOeqvVGHsAXsJxsT28ttosFahLJKCLOTUBqRAtKnVgGSRitds9T432QiT8b77Yga7JIBkulIRRlJPtYhRA==}
+  '@tailwindcss/oxide-wasm32-wasi@4.3.3':
+    resolution: {integrity: sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==}
     engines: {node: '>=14.0.0'}
     cpu: [wasm32]
     bundledDependencies:
@@ -730,24 +739,24 @@ packages:
       - '@emnapi/wasi-threads'
       - tslib
 
-  '@tailwindcss/oxide-win32-arm64-msvc@4.3.1':
-    resolution: {integrity: sha512-aiNvSq9BsVk8V513lDKlrCFAgf8qBMPZTpgEhInL+NwQqs97mYmupVMrPrgBBSL8Pv/0zXu9MrMF9rMun1ZeNg==}
+  '@tailwindcss/oxide-win32-arm64-msvc@4.3.3':
+    resolution: {integrity: sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==}
     engines: {node: '>= 20'}
     cpu: [arm64]
     os: [win32]
 
-  '@tailwindcss/oxide-win32-x64-msvc@4.3.1':
-    resolution: {integrity: sha512-xDEyu1rg290472FEGaKHnzyDyh5QH+AlWvsU5hMoMtPpzmKlRI0jaYKCgSHDYtaQWZOYbMaduSyCwFwY4n1HmA==}
+  '@tailwindcss/oxide-win32-x64-msvc@4.3.3':
+    resolution: {integrity: sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==}
     engines: {node: '>= 20'}
     cpu: [x64]
     os: [win32]
 
-  '@tailwindcss/oxide@4.3.1':
-    resolution: {integrity: sha512-yVPyo8RNkabVr3O2EhHEE0Rewu7YKzc1DhIqfL46LKveFrmu9XbDazNOJY7/GRuvw1h6u3utWnR29H/p5JPlgA==}
+  '@tailwindcss/oxide@4.3.3':
+    resolution: {integrity: sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==}
     engines: {node: '>= 20'}
 
-  '@tailwindcss/vite@4.3.1':
-    resolution: {integrity: sha512-hItDHuIIlEV61R+faXu66s1K36aTurO/Qw0e45Vskz57gXl9pWOT6eg3zmcEui6CZXddbN7zd41bwmvag4JGwQ==}
+  '@tailwindcss/vite@4.3.3':
+    resolution: {integrity: sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==}
     peerDependencies:
       vite: ^5.2.0 || ^6 || ^7 || ^8
 
@@ -793,63 +802,63 @@ packages:
   '@types/web-bluetooth@0.0.21':
     resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==}
 
-  '@typescript-eslint/eslint-plugin@8.61.1':
-    resolution: {integrity: sha512-ZPlVl3PB3et/59Ne0fv/sci6ZXz4T4Hp4nTJ56i/Y0gR89ARb+KphojTq6j+56E5PIezmOIOOWyY+aWQFd+IkQ==}
+  '@typescript-eslint/eslint-plugin@8.65.0':
+    resolution: {integrity: sha512-IEgob78X12rHpUmtcwFsXhZdVGJtwTVP8FiCLZkR6GlYVrl2PcuB+KhCE5BlVC/eQpQnu8WXRtkHZuPar+gCRA==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
-      '@typescript-eslint/parser': ^8.61.1
+      '@typescript-eslint/parser': ^8.65.0
       eslint: ^8.57.0 || ^9.0.0 || ^10.0.0
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/parser@8.61.1':
-    resolution: {integrity: sha512-PJ5vePq5/ognBbrIcoC5+SHO5dfpeLPzP9FpLkzWrguoYQEeeSjlJpVwOpo1JRSTEi7dRcwNy4h4dzV70PqHcg==}
+  '@typescript-eslint/parser@8.65.0':
+    resolution: {integrity: sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       eslint: ^8.57.0 || ^9.0.0 || ^10.0.0
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/project-service@8.61.1':
-    resolution: {integrity: sha512-PrC4JYGmR241lYnfhmKGTXkFqv8+ymbTFgSAY0fVXpY82/QkMw5TZPl+vGzuDDU2QYJk9fIDOBTntF+yDv9LEA==}
+  '@typescript-eslint/project-service@8.65.0':
+    resolution: {integrity: sha512-SxnPhbTsGahizDgbu7oqFH/xVtzIqMd/s+WtnSxNxJZJpLbdT5IPdzg8EZxO3+PoKahXmwJLeNQOpKJb3/bi7Q==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/scope-manager@8.61.1':
-    resolution: {integrity: sha512-L2bdIeoQS8FlKAvONAr20w6OcLXeB+qiDKbAooS9A0Ben+iSIkBef0FxqwKWYqt5sa0i4KJtxVyVmhMylKzF5w==}
+  '@typescript-eslint/scope-manager@8.65.0':
+    resolution: {integrity: sha512-Esbl8OSYiVxBokYgWPf7VVWg/BE798wXhimnn9ML9Pt5qoDf8bfQlgjlKXR/k98+AcNzlLKYrpCcrcuZ9DZLgg==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
 
-  '@typescript-eslint/tsconfig-utils@8.61.1':
-    resolution: {integrity: sha512-UN/H4di+OO7EWx2ovME+8t31YO+KVnK0RRKEHR3kOt21/Ay8BOq3M1OMvWs5vNiqcFCYGYoxK3MXPZzmMUE+yg==}
+  '@typescript-eslint/tsconfig-utils@8.65.0':
+    resolution: {integrity: sha512-j6GzGqCiRdA7Qhur2VVmKZAkBLfnHFQfx4TaJGL9RMveZqCo48jSHHO0DTgizEnGhtWnqmbtCUSrqSkdiY/0Hg==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/type-utils@8.61.1':
-    resolution: {integrity: sha512-GYRicKmVK0C4fsKgaACaknOUAq9Oa2kwsjnpFhFcS/5p4Ht5IP9OVLbgIgcK4SRk92nVHFluurg1lumD9dBcLw==}
+  '@typescript-eslint/type-utils@8.65.0':
+    resolution: {integrity: sha512-YjaZ7PRI5qY7ax2L3PbvX0rRyGtipAReCWs0mhhDBHjH/vl0g0BonaGXrKdKpMbIIsMIwDgbk/xzkBTyAltS5g==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       eslint: ^8.57.0 || ^9.0.0 || ^10.0.0
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/types@8.61.1':
-    resolution: {integrity: sha512-G+CRlPqLv7Bz1IZVs03x5K59F1veqL0EJUROAdGhKsEq8qOiRiZbI+HUojPq5l0fEGOKModD9br6lObhB8zkoA==}
+  '@typescript-eslint/types@8.65.0':
+    resolution: {integrity: sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
 
-  '@typescript-eslint/typescript-estree@8.61.1':
-    resolution: {integrity: sha512-u+oQD3BqYWPc8YV9Zab4vaJElJuwOLPRc10Jm1o/qS+6Qwen14HCWwx0Seo4LnSn2wxea2Ik8DxPt2/FHmuhrg==}
+  '@typescript-eslint/typescript-estree@8.65.0':
+    resolution: {integrity: sha512-JboAE2swaYt4tb1fHhHTABE2K+OLy09XfcTbhnk4Pw96f9dd2e9iYsJ28gBggHlo5z5x1rkyWvcPoTuNTd4oGg==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/utils@8.61.1':
-    resolution: {integrity: sha512-1+P/3Dj6jvtybE1q0HQ6yBt/gq+oKJyLdEv4HdnqasaEXRSYCAsD59mXEVQnM/ULNdQxbX77tdG4jPRjIS6knA==}
+  '@typescript-eslint/utils@8.65.0':
+    resolution: {integrity: sha512-gXiwIHsYreboxeJucHKPvgwl7dXt50mF8s1/c00cP/WoVTyWKFdtfhRWwZiXYFU5H2O8vVoSLNrexFZjYS/SGA==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       eslint: ^8.57.0 || ^9.0.0 || ^10.0.0
       typescript: '>=4.8.4 <6.1.0'
 
-  '@typescript-eslint/visitor-keys@8.61.1':
-    resolution: {integrity: sha512-6fJ9MHWtK14C1DSkiMlHUSOmrVebL7150xZJBlJiL62jjhIA4JmOq6flwBgDxIdBKKdoiZRel+dfPD5MLfny3w==}
+  '@typescript-eslint/visitor-keys@8.65.0':
+    resolution: {integrity: sha512-8C71BQkGjiMmXtop7pHVJu1l2NNShFdkCyD6a2ezzs5vU/L3LRtb69EtcteFwz0mYMPzIgOw0n6OV4VBUWZd7A==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
 
   '@ungap/structured-clone@1.3.0':
@@ -875,17 +884,23 @@ packages:
   '@volar/typescript@2.4.28':
     resolution: {integrity: sha512-Ja6yvWrbis2QtN4ClAKreeUZPVYMARDYZl9LMEv1iQ1QdepB6wn0jTRxA9MftYmYa4DQ4k/DaSZpFPUfxl8giw==}
 
-  '@vue/compiler-core@3.5.38':
-    resolution: {integrity: sha512-s99aGxWYig9ErHbct27KXEGhrBYlRI6c4MwAgXErOAbX9xiW37/uMa+XUDO69zLz83dng8UUZ70CTOJrLrYrEQ==}
+  '@vue/compiler-core@3.5.40':
+    resolution: {integrity: sha512-39E8IgOhTbVDnoJFMKc2DvYnypcZwUqgUhQkccva/0m6FUwtIKSGV7n1hpVmYcFaoRAwf9pBcwnKlCEsN63ZEQ==}
 
-  '@vue/compiler-dom@3.5.38':
-    resolution: {integrity: sha512-JTqp25l8aFfJYF7/KmsXZjAxJz7T+SjmTJLoXVjHtc2BrSgSiW2n9Aem/cWq1OPe68A8JL06B3eVdhlP0H4TVw==}
+  '@vue/compiler-core@3.5.41':
+    resolution: {integrity: sha512-q0Xtv/F9w2YO/7htQhtiL+Ev2WCJbe5N2hc+XfgyKkEKqWpSxknmT8QOuGdEKNdjPq0c3F7rNpFkTo3Kfrm7pg==}
 
-  '@vue/compiler-sfc@3.5.38':
-    resolution: {integrity: sha512-DuA2GiZawSEW442iw/9+Fkol8hTgb4Ke5KkhmSry65QA7YuyMbIdy8p0XZRMvNwJdgRz307W8g1CSzdvS4nuNg==}
+  '@vue/compiler-dom@3.5.40':
+    resolution: {integrity: sha512-pwkx4vqlqOspFstrcmzwkKLePVMD3PT65imRzLhanU2V1Fj4K13g6OXjanOyzw3aTAuRk84BOmY8f3rEHqPaVA==}
 
-  '@vue/compiler-ssr@3.5.38':
-    resolution: {integrity: sha512-7s+W5Gc42FGxZMcuwl8H5B29T8BJPMdBT7KHFE+BbAuZ/iTEdTtv7z2XiMjiaUUw4w3ZcCEdHs36RuYJ2VA7bA==}
+  '@vue/compiler-dom@3.5.41':
+    resolution: {integrity: sha512-oKacVfNglLvGjnS6BXOlGL7EyG2h8X03pqXCjzotRZUaXGjbrTJUnVAQjrCqUnS+lyu31nwQjZY/d817GmCnfw==}
+
+  '@vue/compiler-sfc@3.5.40':
+    resolution: {integrity: sha512-gIf497P4kpuALcvs5n3AEg1Vdn0pSY4XbjASIfHNYF1/MP3T2Mf2STERTubysBxCRxzJGJYtF/O7vwJrxFB3Vw==}
+
+  '@vue/compiler-ssr@3.5.40':
+    resolution: {integrity: sha512-rrE5xiXG663+vHCHa3J9p2z5OcBRjXmoqenprJxAFQxg5pSshzeBiCE6pu46axapRJ2Adk0YDA2BRZVjiHXnhg==}
 
   '@vue/devtools-api@7.7.9':
     resolution: {integrity: sha512-kIE8wvwlcZ6TJTbNeU2HQNtaxLx3a84aotTITUuL/4bzfPxzajGBOoqjMhwZJ8L9qFYDU/lAYMEEm11dnZOD6g==}
@@ -896,28 +911,29 @@ packages:
   '@vue/devtools-shared@7.7.9':
     resolution: {integrity: sha512-iWAb0v2WYf0QWmxCGy0seZNDPdO3Sp5+u78ORnyeonS6MT4PC7VPrryX2BpMJrwlDeaZ6BD4vP4XKjK0SZqaeA==}
 
-  '@vue/language-core@3.3.5':
-    resolution: {integrity: sha512-UkKu5nhX89fg4VhlG/FOeI10G3cj/7radKT/cy9BT4Q9qJmJlSTAc/dP63Xqs29aypN4f39xUV6PsLNk/dcD6g==}
+  '@vue/language-core@3.3.9':
+    resolution: {integrity: sha512-in/68oAa4BCtVY6n/nkuhLIkV8DHYd2UivedJ6cMZ6UYtlq9jaoaSNUBHYCVO44z3nKg7MdE5OBoHKt5SxeBKQ==}
 
-  '@vue/reactivity@3.5.38':
-    resolution: {integrity: sha512-pG6LV/NDNRbKizcUjFFLAfjaL8mcv4DmR9avNcUw2gDHBzZneuS2TWCmp633ynzxz9YYKNeEPK2I8Wraqy2HUQ==}
+  '@vue/reactivity@3.5.40':
+    resolution: {integrity: sha512-B7ot9UlUZOi1zbq61/LvE88ZLTV8IlajTdiZTAEiDQgrnIMIZoPr9kGw0Zw46ObW62O9+H/Be3kMbfb7kYPQZA==}
 
-  '@vue/runtime-core@3.5.38':
-    resolution: {integrity: sha512-iyW8WVfF1CpCXxncZY5Ei6rSd6oZr5DgEom//fUjRBRl56AXPD+s9ATvukRt77ZFTuYlnVA1bxY+dJB94tWVYw==}
+  '@vue/runtime-core@3.5.40':
+    resolution: {integrity: sha512-KAZLweuZ6uUJPK1PMSQPgBU5gCjgrrfjUhSglmU9NhH+Zjepa8cnwSydPWDWHDwOgY4g3VcZ+PljbiHlURNCbw==}
 
-  '@vue/runtime-dom@3.5.38':
-    resolution: {integrity: sha512-apX2wt9sdfDshS+a2xueFZLVpt0GkRJZSoPmrW/SA4yzXTznhfcMVW59gr7h4YQeY0vJhdJkk2rsIDwgfFgC5A==}
+  '@vue/runtime-dom@3.5.40':
+    resolution: {integrity: sha512-ZfrX8ssZQds900L9pr8AuK05ddnMsR4MPMZr8cPN9GoqoPWcXLhjvvbIA2SMv+7a97sJ1vv9pj/zxK0Cq/eEFQ==}
 
-  '@vue/server-renderer@3.5.38':
-    resolution: {integrity: sha512-vue8vbf2QlV4quHqzwmJy6dWfmRhP1J8l4wtZg60CL6VoKqcPY2oe7may3+1d9qfpedjK5PRLFqd5k3Isj9mUw==}
-    peerDependencies:
-      vue: 3.5.38
+  '@vue/server-renderer@3.5.40':
+    resolution: {integrity: sha512-XNJym9WpevhTVt1HuwOrCRJ5Q+9z4BjTMrDtjTrvx74SmUll8spNTw6whWJa9mEkO4PKn5TihI/bm/8ds2QVJw==}
 
   '@vue/shared@3.5.32':
     resolution: {integrity: sha512-ksNyrmRQzWJJ8n3cRDuSF7zNNontuJg1YHnmWRJd2AMu8Ij2bqwiiri2lH5rHtYPZjj4STkNcgcmiQqlOjiYGg==}
 
-  '@vue/shared@3.5.38':
-    resolution: {integrity: sha512-FTW0AFZNaK5/mOqvGBwVfUlNLU38TiQn4+DQgIFUnrBBJQ1crMJ82yeGQLV5jyKFsO8yRukpbuP7x+nRbH6aug==}
+  '@vue/shared@3.5.40':
+    resolution: {integrity: sha512-WxnBtruIqOoV3rA4jeKDWzrYI5h7Cp4+pjwDi8kWGHz+IslhiN+wguLVVhtv2l8VoU02rzDCVfDjgCl1lNpZVg==}
+
+  '@vue/shared@3.5.41':
+    resolution: {integrity: sha512-IOnwSCma8j+9xJT6b8H0dEYidC80NsYmNMlZxRsukYcSoGaDBohog5hDxzeUXdFeGWFA++vWvxqOmrr96VlqMA==}
 
   '@vueuse/core@12.8.2':
     resolution: {integrity: sha512-HbvCmZdzAu3VGi/pWYm5Ut+Kd9mn1ZHnn4L5G8kOQTPs/IwIAmJoBrmYk2ckLArgMXZj0AW3n5CAejLUO+PhdQ==}
@@ -974,8 +990,8 @@ packages:
     peerDependencies:
       acorn: ^6.0.0 || ^7.0.0 || ^8.0.0
 
-  acorn@8.17.0:
-    resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==}
+  acorn@8.18.0:
+    resolution: {integrity: sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==}
     engines: {node: '>=0.4.0'}
     hasBin: true
 
@@ -995,8 +1011,8 @@ packages:
   argparse@2.0.1:
     resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
 
-  autoprefixer@10.5.0:
-    resolution: {integrity: sha512-FMhOoZV4+qR6aTUALKX2rEqGG+oyATvwBt9IIzVR5rMa2HRWPkxf+P+PAJLD1I/H5/II+HuZcBJYEFBpq39ong==}
+  autoprefixer@10.5.4:
+    resolution: {integrity: sha512-MaU0U/za7N3r6brxD4YB/l4NSrFzLPlANv6wEuQVaIPlD3L4W9rFcQPbL/EilY9BHhHvhfcz3gInDLrEtWT4EA==}
     engines: {node: ^10 || ^12 || >=14}
     hasBin: true
     peerDependencies:
@@ -1006,8 +1022,8 @@ packages:
     resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==}
     engines: {node: 18 || 20 || >=22}
 
-  baseline-browser-mapping@2.10.20:
-    resolution: {integrity: sha512-1AaXxEPfXT+GvTBJFuy4yXVHWJBXa4OdbIebGN/wX5DlsIkU0+wzGnd2lOzokSk51d5LUmqjgBLRLlypLUqInQ==}
+  baseline-browser-mapping@2.11.1:
+    resolution: {integrity: sha512-HYXq73DDpCtNzOmrFsm9eSwCvWCql0RzqjpDzXN9EadiLJ4DNat0nsZ/Bzmy+Ud12mb4/zKDY0cQ805ZzN+i0A==}
     engines: {node: '>=6.0.0'}
     hasBin: true
 
@@ -1017,21 +1033,21 @@ packages:
   boolbase@1.0.0:
     resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
 
-  brace-expansion@5.0.6:
-    resolution: {integrity: sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==}
-    engines: {node: 18 || 20 || >=22}
+  brace-expansion@5.0.8:
+    resolution: {integrity: sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==}
+    engines: {node: 20 || >=22}
 
   braces@3.0.3:
     resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
     engines: {node: '>=8'}
 
-  browserslist@4.28.2:
-    resolution: {integrity: sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==}
+  browserslist@4.28.7:
+    resolution: {integrity: sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==}
     engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}
     hasBin: true
 
-  caniuse-lite@1.0.30001788:
-    resolution: {integrity: sha512-6q8HFp+lOQtcf7wBK+uEenxymVWkGKkjFpCvw5W25cmMwEDU45p1xQFBQv8JDlMMry7eNxyBaR+qxgmTUZkIRQ==}
+  caniuse-lite@1.0.30001806:
+    resolution: {integrity: sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==}
 
   ccount@2.0.1:
     resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
@@ -1211,14 +1227,14 @@ packages:
   domutils@3.2.2:
     resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}
 
-  electron-to-chromium@1.5.340:
-    resolution: {integrity: sha512-908qahOGocRMinT2nM3ajCEM99H4iPdv84eagPP3FfZy/1ZGeOy2CZYzjhms81ckOPCXPlW7LkY4XpxD8r1DrA==}
+  electron-to-chromium@1.5.395:
+    resolution: {integrity: sha512-7zt9Aw+SrmxLWLN0zhaTWZQiCdryLVrYTq5R7iZakLvi2UQPYMMsROYV/2qVCzMeCiSXHwKOU+sZ4zOVVlrtKA==}
 
   emoji-regex-xs@1.0.0:
     resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==}
 
-  enhanced-resolve@5.21.6:
-    resolution: {integrity: sha512-aNnGCvbJ/RIyWo1IuhNdVjnNF+EjH9wpzpNHt+ci/m9He9LJvUN8wrCcXjp9cWsGNAuvSpVFTx/vraAFQ8qGjQ==}
+  enhanced-resolve@5.24.3:
+    resolution: {integrity: sha512-PwKooW9JUzh5chmYfHM3IQl5OkK2u2Nm011MgeZrss3JmFraUx/fqrf78kk8GUMYoibx/14MdwTl/1WKkG7TpQ==}
     engines: {node: '>=10.13.0'}
 
   entities@4.5.0:
@@ -1248,8 +1264,8 @@ packages:
     peerDependencies:
       eslint: '>=7.0.0'
 
-  eslint-plugin-vue@10.9.2:
-    resolution: {integrity: sha512-4g7ZP3pYcuqd7Zp0pzUKcos0W+RkjBz4EGdhJ92FcYk6v03Ti/GK5NwjgsjxHK+98eXDbHeK7VtX1az7/8doZA==}
+  eslint-plugin-vue@10.10.0:
+    resolution: {integrity: sha512-dL9x9rBHqqNcByWiLOHK6L0SB97V82/NC0cZRn9cXPjM7pCuWlpQQP9bFH4vjBv80ej1ZpzAkuD8zWH1o9bZbA==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       '@stylistic/eslint-plugin': ^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0
@@ -1274,8 +1290,8 @@ packages:
     resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==}
     engines: {node: ^20.19.0 || ^22.13.0 || >=24}
 
-  eslint@10.5.0:
-    resolution: {integrity: sha512-1y+7C+vi12bUK1IpZeaV3gsH9fHLBmPvYmPx42pvT/E9yG0IC8g3PUZZgp0+JLJl7ZDK0flc2gc+Aw9dpCvIsQ==}
+  eslint@10.8.0:
+    resolution: {integrity: sha512-nuKKvN+oIBO0koN7Tm7dlkmnkc21mtt0QJLwAKzjLq14y6lRTdVG36MZHJ8eQHwdJMwZbQNMlPOYedMq/oVJvQ==}
     engines: {node: ^20.19.0 || ^22.13.0 || >=24}
     hasBin: true
     peerDependencies:
@@ -1357,8 +1373,8 @@ packages:
     resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==}
     engines: {node: '>=16'}
 
-  flatted@3.4.2:
-    resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==}
+  flatted@3.4.3:
+    resolution: {integrity: sha512-/zipXxyO6rGvuNGDiULY9MvEGSkb2gaG4GGH4ygMi0ZZzyMHdUZBmntJmx5x1G2VuPytCwGN4xsJP6cw+sK+vQ==}
 
   focus-trap@7.8.0:
     resolution: {integrity: sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==}
@@ -1406,8 +1422,8 @@ packages:
     resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
     engines: {node: '>= 4'}
 
-  ignore@7.0.5:
-    resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==}
+  ignore@7.0.6:
+    resolution: {integrity: sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==}
     engines: {node: '>= 4'}
 
   imurmurhash@0.1.4:
@@ -1551,8 +1567,8 @@ packages:
     resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==}
     engines: {node: '>= 12.0.0'}
 
-  linkify-it@5.0.1:
-    resolution: {integrity: sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==}
+  linkify-it@5.0.2:
+    resolution: {integrity: sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==}
 
   locate-path@6.0.0:
     resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
@@ -1570,8 +1586,8 @@ packages:
   markdown-it-footnote@4.0.0:
     resolution: {integrity: sha512-WYJ7urf+khJYl3DqofQpYfEYkZKbmXmwxQV8c8mO/hGIhgZ1wOe7R4HLFNwqx7TjILbnC98fuyeSsin19JdFcQ==}
 
-  markdown-it@14.2.0:
-    resolution: {integrity: sha512-1TGiQiJVRQ3NPmZH6sx5Cfnmg6GQm9jvC1ch4TK511NjSJvjzKLzn5pPfZRNZkRPZP0HqCioSndqH8v2nRaWVQ==}
+  markdown-it@14.3.0:
+    resolution: {integrity: sha512-RCEsPjR+sr0x+AuYp601tKTkgFG4YEPLCzHST3cQ/fhlJkqAkz1L2/Qbp1j9qw5SBwQHFBoW8+hoN5xssOF0Tw==}
     hasBin: true
 
   mdast-util-to-hast@13.2.1:
@@ -1609,8 +1625,8 @@ packages:
     resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==}
     engines: {node: '>=8.6'}
 
-  minimatch@10.2.5:
-    resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==}
+  minimatch@10.2.6:
+    resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==}
     engines: {node: 18 || 20 || >=22}
 
   minisearch@7.2.0:
@@ -1633,6 +1649,11 @@ packages:
     engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
     hasBin: true
 
+  nanoid@3.3.16:
+    resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==}
+    engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
+    hasBin: true
+
   natural-compare@1.4.0:
     resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
 
@@ -1640,8 +1661,9 @@ packages:
     resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==}
     hasBin: true
 
-  node-releases@2.0.37:
-    resolution: {integrity: sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==}
+  node-releases@2.0.51:
+    resolution: {integrity: sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==}
+    engines: {node: '>=18'}
 
   nth-check@2.1.1:
     resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
@@ -1682,12 +1704,12 @@ packages:
     resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==}
     engines: {node: '>=8.6'}
 
-  picomatch@4.0.4:
-    resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==}
+  picomatch@4.0.5:
+    resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==}
     engines: {node: '>=12'}
 
-  postcss-selector-parser@7.1.1:
-    resolution: {integrity: sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==}
+  postcss-selector-parser@7.1.4:
+    resolution: {integrity: sha512-HeP7D2wyhkR+XaK6v4W8oRF62Dsz4flyuczALJp61GckGm42u1saSSJ/0auvcBqxs3jMRFEcPK34At/0JBKdOg==}
     engines: {node: '>=4'}
 
   postcss-value-parser@4.2.0:
@@ -1697,8 +1719,8 @@ packages:
     resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==}
     engines: {node: ^10 || ^12 || >=14}
 
-  postcss@8.5.15:
-    resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==}
+  postcss@8.5.22:
+    resolution: {integrity: sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==}
     engines: {node: ^10 || ^12 || >=14}
 
   preact@10.29.1:
@@ -1708,8 +1730,8 @@ packages:
     resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
     engines: {node: '>= 0.8.0'}
 
-  prettier-plugin-tailwindcss@0.8.0:
-    resolution: {integrity: sha512-V8ITGH87yuBDF6JpEZTOVlUz/saAwqb8f3HRgUj8Lh+tGCcrmorhsLpYqzygwFwK0PE2Ib6Mv3M7T/uE2tZV1g==}
+  prettier-plugin-tailwindcss@0.8.1:
+    resolution: {integrity: sha512-iaFMYqDsE4ffdDkn5qup0j5f2aCEBFZrdrZnvu9QKTlWx/iGPeQ4HHu7b7fCPMxeo9nwQBiOAh2nSypdFYWJkw==}
     engines: {node: '>=20.19'}
     peerDependencies:
       '@ianvs/prettier-plugin-sort-imports': '*'
@@ -1768,8 +1790,8 @@ packages:
     peerDependencies:
       prettier: ^3.5.3
 
-  prettier@3.8.4:
-    resolution: {integrity: sha512-N2MylSdi48+5N/6S5j+maeHbUSIzzZ5uOcX5Hm4QpV8Dkb1HFjfAKTKX6yNPJQD9AhcT3ifHNB66tWTTJDi11Q==}
+  prettier@3.9.6:
+    resolution: {integrity: sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==}
     engines: {node: '>=14'}
     hasBin: true
 
@@ -1852,13 +1874,8 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
 
-  semver@7.8.3:
-    resolution: {integrity: sha512-wnilbGyMxzbY7dNOl7jpKbLSjcfeweJWU5j4+u5qW+6/wuGD9KzIGOyZnQVSBM9E7DtWaaH3CyHkppYrKYoxwg==}
-    engines: {node: '>=10'}
-    hasBin: true
-
-  semver@7.8.4:
-    resolution: {integrity: sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==}
+  semver@7.8.5:
+    resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==}
     engines: {node: '>=10'}
     hasBin: true
 
@@ -1887,8 +1904,8 @@ packages:
   sprintf-js@1.0.3:
     resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
 
-  sql-formatter@15.8.1:
-    resolution: {integrity: sha512-nT2r90kTEYBuse9fe4r1Rp78v1mOBD35KsGc07Vo9eQSVa1TcTSnCS0zouf6BCmdzvmqBsBW+cYuBoYkHO/OWg==}
+  sql-formatter@15.8.2:
+    resolution: {integrity: sha512-kTYRg5FIcvsDtYUG2Qn9pYT6xKwiLJN5TTIvc5Mur6hIg4pSfdpHu8Yyu5bqESLHnVM3mXzD446cb2+uEaKZXg==}
     hasBin: true
 
   stringify-entities@4.0.4:
@@ -1902,16 +1919,16 @@ packages:
     resolution: {integrity: sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==}
     engines: {node: '>=16'}
 
-  svgo@4.0.1:
-    resolution: {integrity: sha512-XDpWUOPC6FEibaLzjfe0ucaV0YrOjYotGJO1WpF0Zd+n6ZGEQUsSugaoLq9QkEZtAfQIxT42UChcssDVPP3+/w==}
+  svgo@4.0.2:
+    resolution: {integrity: sha512-ekx94z1rRc5LDi6oSUaeRnYhd0UOJxdtQCL2rF8xpWxD3TPAsISWOrxezqGovqS38GRZOdpDfvQe3ts6F7nsng==}
     engines: {node: '>=16'}
     hasBin: true
 
   tabbable@6.4.0:
     resolution: {integrity: sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==}
 
-  tailwindcss@4.3.1:
-    resolution: {integrity: sha512-hk+TB1m+K8CYNrP6rjQaq/Y+4Zylwpa87mLYBKCunwnnQ9p+fHb7kmSfGqyEJoxF/O6CDyABWVFEafNSYKll+Q==}
+  tailwindcss@4.3.3:
+    resolution: {integrity: sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==}
 
   tapable@2.3.3:
     resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==}
@@ -1942,8 +1959,8 @@ packages:
     resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==}
     engines: {node: '>= 0.8.0'}
 
-  typescript-eslint@8.61.1:
-    resolution: {integrity: sha512-V7PayAfJokV3pEHgN7/v03D1SpujhRfQtYLbLIiBfDDncdg4PAiRBfoS4cnCANK4jmAPncczi59QO3afiXUlNw==}
+  typescript-eslint@8.65.0:
+    resolution: {integrity: sha512-/ggrHAwyjENDusvyxbuqxAC2dTnZg/Z8F+fgQtYIz+L6n/9HfSlEZcFGV/NsMNa6CkGk0xUjUAFwC0vHOflvIA==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
     peerDependencies:
       eslint: ^8.57.0 || ^9.0.0 || ^10.0.0
@@ -1990,92 +2007,92 @@ packages:
   vega-canvas@2.0.0:
     resolution: {integrity: sha512-9x+4TTw/USYST5nx4yN272sy9WcqSRjAR0tkQYZJ4cQIeon7uVsnohvoPQK1JZu7K1QXGUqzj08z0u/UegBVMA==}
 
-  vega-crossfilter@5.1.0:
-    resolution: {integrity: sha512-EmVhfP3p6AM7o/lPan/QAoqjblI19BxWUlvl2TSs0xjQd8KbaYYbS4Ixt3cmEvl0QjRdBMF6CdJJ/cy9DTS4Fw==}
+  vega-crossfilter@5.1.2:
+    resolution: {integrity: sha512-K14PtfBxQzb7UJq5rL1IuwDdPov6lucoHVzRaraGiBBnfc963jkWHQNfRFAuEj/nIt5BIixXBiqIrGKgfScjSA==}
 
-  vega-dataflow@6.1.0:
-    resolution: {integrity: sha512-JxumGlODtFbzoQ4c/jQK8Tb/68ih0lrexlCozcMfTAwQ12XhTqCvlafh7MAKKTMBizjOfaQTHm4Jkyb1H5CfyQ==}
+  vega-dataflow@6.1.2:
+    resolution: {integrity: sha512-0/HBBlzPERh0wV5kUOh/N4aAAmb3rB18Kfpej+nWvDIjPk3dixTapNkVVkoA9CsB5dY5bY+HNqiM/Pp4XPduGQ==}
 
-  vega-encode@5.1.0:
-    resolution: {integrity: sha512-q26oI7B+MBQYcTQcr5/c1AMsX3FvjZLQOBi7yI0vV+GEn93fElDgvhQiYrgeYSD4Exi/jBPeUXuN6p4bLz16kA==}
+  vega-encode@5.2.1:
+    resolution: {integrity: sha512-YuvJ66z6FQRtu0n0IYxsJp7BpliXVgaxNsur0Fqic10/a5TLMQmDYURPUcrtWViAMpqKgdNMyxnmr0ZQ4bjwCA==}
 
   vega-event-selector@4.0.0:
     resolution: {integrity: sha512-CcWF4m4KL/al1Oa5qSzZ5R776q8lRxCj3IafCHs5xipoEHrkgu1BWa7F/IH5HrDNXeIDnqOpSV1pFsAWRak4gQ==}
 
-  vega-expression@6.1.0:
-    resolution: {integrity: sha512-hHgNx/fQ1Vn1u6vHSamH7lRMsOa/yQeHGGcWVmh8fZafLdwdhCM91kZD9p7+AleNpgwiwzfGogtpATFaMmDFYg==}
+  vega-expression@6.2.1:
+    resolution: {integrity: sha512-AK4EYh9ErCC7KU8vGigsetwG66S1wLLfPFnfjkOdydRaN4kNJp129qp4jos0wA/r7F/lIlhM3uIzw+KSA2cZEg==}
 
-  vega-force@5.1.0:
-    resolution: {integrity: sha512-wdnchOSeXpF9Xx8Yp0s6Do9F7YkFeOn/E/nENtsI7NOcyHpICJ5+UkgjUo9QaQ/Yu+dIDU+sP/4NXsUtq6SMaQ==}
+  vega-force@5.1.2:
+    resolution: {integrity: sha512-693paaBvACvAtUO8vp4m/KUJ0F3tKaoXixzw+ExpP4qKkqJ/Me2MGWsVrEJh/f6KuU9bp7xv+g7OPNhqnVrpoQ==}
 
-  vega-format@2.1.0:
-    resolution: {integrity: sha512-i9Ht33IgqG36+S1gFDpAiKvXCPz+q+1vDhDGKK8YsgMxGOG4PzinKakI66xd7SdV4q97FgpR7odAXqtDN2wKqw==}
+  vega-format@2.1.2:
+    resolution: {integrity: sha512-cJ2oqGEGYJ2NbkIIUDic03ZZX1meE+sq330gq4HFf3CvzSupTwHLWQj3/H5zug5DrqEQhLCLkiRwvmMe4pEqfg==}
 
-  vega-functions@6.1.1:
-    resolution: {integrity: sha512-Due6jP0y0FfsGMTrHnzUGnEwXPu7VwE+9relfo+LjL/tRPYnnKqwWvzt7n9JkeBuZqjkgYjMzm/WucNn6Hkw5A==}
+  vega-functions@6.1.3:
+    resolution: {integrity: sha512-WOgWVxGg1T1tuICyylIppxXWXWEtl7P81Xnj0vQtoIn11VvQ0mrsBZr1aia4g8Sw0fEH1mUO7Dtj7v3o8BgBmQ==}
 
-  vega-geo@5.1.0:
-    resolution: {integrity: sha512-H8aBBHfthc3rzDbz/Th18+Nvp00J73q3uXGAPDQqizioDm/CoXCK8cX4pMePydBY9S6ikBiGJrLKFDa80wI20g==}
+  vega-geo@5.1.2:
+    resolution: {integrity: sha512-tls2+IVuKaIOxEqdshuSk2Til8qcF7SUNaWkATKu5hwWqQEY3cp7tfdXosuTMaDTkLp+Nw84iWfe5BzfoxDYMA==}
 
-  vega-hierarchy@5.1.0:
-    resolution: {integrity: sha512-rZlU8QJNETlB6o73lGCPybZtw2fBBsRIRuFE77aCLFHdGsh6wIifhplVarqE9icBqjUHRRUOmcEYfzwVIPr65g==}
+  vega-hierarchy@5.1.2:
+    resolution: {integrity: sha512-7a72/lA6j50HU6PQe5ChbomftHi29V1oCcibaQpDozZncHuAnsQummqfiPyTMPpNMrSRtyZy18do0IEoOvrbMg==}
 
-  vega-label@2.1.0:
-    resolution: {integrity: sha512-/hgf+zoA3FViDBehrQT42Lta3t8In6YwtMnwjYlh72zNn1p3c7E3YUBwqmAqTM1x+tudgzMRGLYig+bX1ewZxQ==}
+  vega-label@2.1.2:
+    resolution: {integrity: sha512-MFsOKOCG6a5QwP5atoFFi+R58IP/Q7//65UvgrSeankX2QrDcWjBtzIqS6JlYBOAdyjoiAAghx7pnIAj5nAO6A==}
 
-  vega-loader@5.1.0:
-    resolution: {integrity: sha512-GaY3BdSPbPNdtrBz8SYUBNmNd8mdPc3mtdZfdkFazQ0RD9m+Toz5oR8fKnTamNSk9fRTJX0Lp3uEqxrAlQVreg==}
+  vega-loader@5.1.2:
+    resolution: {integrity: sha512-O29bKHsSJBi391H88OZXH8vgUdopWMorC5ZJ/8XjTq8TIXVdeUrNnI8dMkUyv7yQg3ywMTji+5fHhov1TOkkyA==}
 
-  vega-parser@7.1.0:
-    resolution: {integrity: sha512-g0lrYxtmYVW8G6yXpIS4J3Uxt9OUSkc0bLu5afoYDo4rZmoOOdll3x3ebActp5LHPW+usZIE+p5nukRS2vEc7Q==}
+  vega-parser@7.1.2:
+    resolution: {integrity: sha512-88HNkVrEwWbMmbbhPCmFHPrl/ClTcvThRv2UMZqMJJ5IqgjWLUolCA1fhAvnAbvRPe+Z1j+GbV/WWjYWAjEuyw==}
 
-  vega-projection@2.1.0:
-    resolution: {integrity: sha512-EjRjVSoMR5ibrU7q8LaOQKP327NcOAM1+eZ+NO4ANvvAutwmbNVTmfA1VpPH+AD0AlBYc39ND/wnRk7SieDiXA==}
+  vega-projection@2.1.2:
+    resolution: {integrity: sha512-HL+FS9AgYSOa1UOIJfhOD39RMT4tDODkMcN6MCRsdlRNnukVm1+JhrixDnciBjqiZLxDhUjESJwgLQtXx8cUyw==}
 
-  vega-regression@2.1.0:
-    resolution: {integrity: sha512-HzC7MuoEwG1rIxRaNTqgcaYF03z/ZxYkQR2D5BN0N45kLnHY1HJXiEcZkcffTsqXdspLjn47yLi44UoCwF5fxQ==}
+  vega-regression@2.1.2:
+    resolution: {integrity: sha512-2dyb2sT6cxdTK5K87uNDYpW7LbTP90lplxW62imjfLAwKUq1wL74X8y03a0W5tpRfV8+f3ai+cYTv+HXB3XmDQ==}
 
-  vega-runtime@7.1.0:
-    resolution: {integrity: sha512-mItI+WHimyEcZlZrQ/zYR3LwHVeyHCWwp7MKaBjkU8EwkSxEEGVceyGUY9X2YuJLiOgkLz/6juYDbMv60pfwYA==}
+  vega-runtime@7.1.2:
+    resolution: {integrity: sha512-MMfm7FeWLX2XqaY26i99GrlToexO/EWYqT7MRqEkpw+0nYgKh4zpQ47j04LB3ishnSZAj2AL0HNl9uPZLxgFhg==}
 
-  vega-scale@8.1.0:
-    resolution: {integrity: sha512-VEgDuEcOec8+C8+FzLcnAmcXrv2gAJKqQifCdQhkgnsLa978vYUgVfCut/mBSMMHbH8wlUV1D0fKZTjRukA1+A==}
+  vega-scale@8.1.2:
+    resolution: {integrity: sha512-ADCGgdSmhcZyHxtHbrVcDrG76er2s2eXowO3qDg0vx2eclHdXL4YNaEjTHrosNG/1TAI5LAe7aQ2BGssJFfAXA==}
 
-  vega-scenegraph@5.1.0:
-    resolution: {integrity: sha512-4gA89CFIxkZX+4Nvl8SZF2MBOqnlj9J5zgdPh/HPx+JOwtzSlUqIhxFpFj7GWYfwzr/PyZnguBLPihPw1Og/cA==}
+  vega-scenegraph@5.2.1:
+    resolution: {integrity: sha512-tWolI3s/cjU5g8evEjpxJqgqKQ2IJv1FoPXqvNuviLNdylP1I/h5iF5RY08Cz4jkuP9zircSskFEBsHylD1dgQ==}
 
-  vega-selections@6.1.2:
-    resolution: {integrity: sha512-xJ+V4qdd46nk2RBdwIRrQm2iSTMHdlu/omhLz1pqRL3jZDrkqNBXimrisci2kIKpH2WBpA1YVagwuZEKBmF2Qw==}
+  vega-selections@6.1.4:
+    resolution: {integrity: sha512-qR5feBL8xq5BjoG3rf7e2WKle/cn5Fa9pBFMBnzoOYp4nTA0GYDoKuOjZkftibsQcmdST0HzPjSrS14jH3TCQw==}
 
   vega-statistics@2.0.0:
     resolution: {integrity: sha512-dGPfDXnBlgXbZF3oxtkb8JfeRXd5TYHx25Z/tIoaa9jWua4Vf/AoW2wwh8J1qmMy8J03/29aowkp1yk4DOPazQ==}
 
-  vega-time@3.1.0:
-    resolution: {integrity: sha512-G93mWzPwNa6UYQRkr8Ujur9uqxbBDjDT/WpXjbDY0yygdSkRT+zXF+Sb4gjhW0nPaqdiwkn0R6kZcSPMj1bMNA==}
+  vega-time@3.2.1:
+    resolution: {integrity: sha512-SXo1klLYPsmBdSj9itYSuFMmCpwnnIvQS8jWFxsCTnyWcPaBQKz0kf2M7DqTysi34KTk+6ws4kqZg11XMWnIYA==}
 
-  vega-transforms@5.1.0:
-    resolution: {integrity: sha512-mj/sO2tSuzzpiXX8JSl4DDlhEmVwM/46MTAzTNQUQzJPMI/n4ChCjr/SdEbfEyzlD4DPm1bjohZGjLc010yuMg==}
+  vega-transforms@5.2.1:
+    resolution: {integrity: sha512-csKiYXJFa0YY5lubP1wc8b99SJ8nHLG/sOWvKFpW3CfDfjp8LoB0exrxgtIoClCsngAguKiuZVDwQZxBhPJZwA==}
 
-  vega-typings@2.1.0:
-    resolution: {integrity: sha512-zdis4Fg4gv37yEvTTSZEVMNhp8hwyEl7GZ4X4HHddRVRKxWFsbyKvZx/YW5Z9Ox4sjxVA2qHzEbod4Fdx+SEJA==}
+  vega-typings@2.2.0:
+    resolution: {integrity: sha512-pX7LsqgMpzMPOyydg/drYbIjh+mmvLfKtuKr75OpCu/ZYKJ2i8UjSmNlpARYeYGLaXfuNUAM3hMN8NPZPDYiBQ==}
 
-  vega-util@2.1.0:
-    resolution: {integrity: sha512-PGfp0m0QCufDmcxKJCWQy4Ov23FoF8DSXmoJwSezi3itQaa2hbxK0+xwsTMP2vy4PR16Pu25HMzgMwXVW1+33w==}
+  vega-util@2.1.2:
+    resolution: {integrity: sha512-Mdvv33BUlx4no5gz2VHdue4bJ6ejWmInuieID3JGnAXwewHKrPWE/Xr2vVWA+xS9Om1eHvj/0wVsiBGJ61v2uQ==}
 
-  vega-view-transforms@5.1.0:
-    resolution: {integrity: sha512-fpigh/xn/32t+An1ShoY3MLeGzNdlbAp2+HvFKzPpmpMTZqJEWkk/J/wHU7Swyc28Ta7W1z3fO+8dZkOYO5TWQ==}
+  vega-view-transforms@5.2.1:
+    resolution: {integrity: sha512-TiguoCJ6BB7TEV+s3nNMoCoH4+q5A6XQsDMEzbVuRaSWmfIJxvWUlbZQqn4CUt65VMDr63Fy21Xpb0ntlaU5LQ==}
 
-  vega-view@6.1.0:
-    resolution: {integrity: sha512-hmHDm/zC65lb23mb9Tr9Gx0wkxP0TMS31LpMPYxIZpvInxvUn7TYitkOtz1elr63k2YZrgmF7ztdGyQ4iCQ5fQ==}
+  vega-view@6.1.2:
+    resolution: {integrity: sha512-BG/Kqd1csdK4uKdJ4hv72msgQX3Ry5SzCV1ngbytkn6QwuQPpFWzya/f0OSWyMQP9S79avdHXqdLCxjSeiDQRw==}
 
-  vega-voronoi@5.1.0:
-    resolution: {integrity: sha512-uKdsoR9x60mz7eYtVG+NhlkdQXeVdMr6jHNAHxs+W+i6kawkUp5S9jp1xf1FmW/uZvtO1eqinHQNwATcDRsiUg==}
+  vega-voronoi@5.1.2:
+    resolution: {integrity: sha512-8BgKb6aF/42L/PrFi3Idqag7FtbzNqDVhvPD4SfswSzGYlRDcxcn1eQQsa//q72qIZk4i5rsHeAiH2QbIjPSBg==}
 
-  vega-wordcloud@5.1.0:
-    resolution: {integrity: sha512-sSdNmT8y2D7xXhM2h76dKyaYn3PA4eV49WUUkfYfqHz/vpcu10GSAoFxLhQQTkbZXR+q5ZB63tFUow9W2IFo6g==}
+  vega-wordcloud@5.1.2:
+    resolution: {integrity: sha512-ZSw6fupf4qx4hBzL9q/oVUKPVbVePjJ/FSgfxxgRcKvE1VoYq3HWfx9qg+CQMvgDS+oGDsn2WoQJuNLYx9T3AQ==}
 
-  vega@6.2.0:
-    resolution: {integrity: sha512-BIwalIcEGysJdQDjeVUmMWB3e50jPDNAMfLJscjEvpunU9bSt7X1OYnQxkg3uBwuRRI4nWfFZO9uIW910nLeGw==}
+  vega@6.3.1:
+    resolution: {integrity: sha512-mX5tvY3ISCSiPPmunuZyQfccq0XlUvJd2t5oyc6SNS0N0TwnPNoklx0mVod5n+qbzuUoiuxl7Ve/SvoRqH4RzA==}
 
   vfile-message@4.0.3:
     resolution: {integrity: sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==}
@@ -2135,14 +2152,14 @@ packages:
     peerDependencies:
       eslint: ^8.57.0 || ^9.0.0 || ^10.0.0
 
-  vue-tsc@3.3.5:
-    resolution: {integrity: sha512-Rzh/G2MmNlMSAMTiQEjDrsb4dgB/jbtEM47rVN2NtidF1dfb/q4w4QvpQBtW5+y3y5H27Hjh7deVwk+YB02fNg==}
+  vue-tsc@3.3.9:
+    resolution: {integrity: sha512-TS3Y1ux/IRoE8OCP2PpACAeOseuIs0UvWrcr7u+w3PmfY+SlCfEf8zjrBgnQksHUgLpthi5vHlffcQTQTdPBZA==}
     hasBin: true
     peerDependencies:
       typescript: '>=5.0.0'
 
-  vue@3.5.38:
-    resolution: {integrity: sha512-vAMKHfImQlYSy0C+PBue4s3ERZ2xGKfgZg5GXAsLInq1dyh2H78ILVP5sK0KPFPVW4kv+OGCIvBEondcjpZp7A==}
+  vue@3.5.40:
+    resolution: {integrity: sha512-+8PJ4SJXdn/cHGImF4CKdxlWHIN5Dkt7DoufRREM6h6uVCx2m7QxgcEQmmzyOK8A9mcafg7sFbJFYsdFVubTig==}
     peerDependencies:
       typescript: '*'
     peerDependenciesMeta:
@@ -2158,9 +2175,9 @@ packages:
     resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
     engines: {node: '>=0.10.0'}
 
-  xml-name-validator@4.0.0:
-    resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==}
-    engines: {node: '>=12'}
+  xml-name-validator@5.0.0:
+    resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}
+    engines: {node: '>=18'}
 
   yocto-queue@0.1.0:
     resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
@@ -2315,6 +2332,10 @@ snapshots:
     dependencies:
       '@babel/types': 7.29.7
 
+  '@babel/parser@7.29.8':
+    dependencies:
+      '@babel/types': 7.29.8
+
   '@babel/template@7.28.6':
     dependencies:
       '@babel/code-frame': 7.29.0
@@ -2343,6 +2364,11 @@ snapshots:
       '@babel/helper-string-parser': 7.29.7
       '@babel/helper-validator-identifier': 7.29.7
 
+  '@babel/types@7.29.8':
+    dependencies:
+      '@babel/helper-string-parser': 7.29.7
+      '@babel/helper-validator-identifier': 7.29.7
+
   '@docsearch/css@3.8.2': {}
 
   '@docsearch/js@3.8.2(@algolia/client-search@5.50.2)(search-insights@2.17.3)':
@@ -2436,9 +2462,9 @@ snapshots:
   '@esbuild/win32-x64@0.21.5':
     optional: true
 
-  '@eslint-community/eslint-utils@4.9.1(eslint@10.5.0(jiti@2.7.0))':
+  '@eslint-community/eslint-utils@4.10.1(eslint@10.8.0(jiti@2.7.0))':
     dependencies:
-      eslint: 10.5.0(jiti@2.7.0)
+      eslint: 10.8.0(jiti@2.7.0)
       eslint-visitor-keys: 3.4.3
 
   '@eslint-community/regexpp@4.12.2': {}
@@ -2447,11 +2473,11 @@ snapshots:
     dependencies:
       '@eslint/object-schema': 3.0.5
       debug: 4.4.3
-      minimatch: 10.2.5
+      minimatch: 10.2.6
     transitivePeerDependencies:
       - supports-color
 
-  '@eslint/config-helpers@0.6.0':
+  '@eslint/config-helpers@0.7.0':
     dependencies:
       '@eslint/core': 1.2.1
 
@@ -2459,9 +2485,9 @@ snapshots:
     dependencies:
       '@types/json-schema': 7.0.15
 
-  '@eslint/js@10.0.1(eslint@10.5.0(jiti@2.7.0))':
+  '@eslint/js@10.0.1(eslint@10.8.0(jiti@2.7.0))':
     optionalDependencies:
-      eslint: 10.5.0(jiti@2.7.0)
+      eslint: 10.8.0(jiti@2.7.0)
 
   '@eslint/object-schema@3.0.5': {}
 
@@ -2486,16 +2512,16 @@ snapshots:
 
   '@humanwhocodes/retry@0.4.3': {}
 
-  '@ianvs/prettier-plugin-sort-imports@4.7.1(@vue/compiler-sfc@3.5.38)(prettier@3.8.4)':
+  '@ianvs/prettier-plugin-sort-imports@4.7.1(@vue/compiler-sfc@3.5.40)(prettier@3.9.6)':
     dependencies:
       '@babel/generator': 7.29.1
       '@babel/parser': 7.29.2
       '@babel/traverse': 7.29.0
       '@babel/types': 7.29.0
-      prettier: 3.8.4
+      prettier: 3.9.6
       semver: 7.7.4
     optionalDependencies:
-      '@vue/compiler-sfc': 3.5.38
+      '@vue/compiler-sfc': 3.5.40
     transitivePeerDependencies:
       - supports-color
 
@@ -2651,72 +2677,72 @@ snapshots:
 
   '@shikijs/vscode-textmate@10.0.2': {}
 
-  '@tailwindcss/node@4.3.1':
+  '@tailwindcss/node@4.3.3':
     dependencies:
       '@jridgewell/remapping': 2.3.5
-      enhanced-resolve: 5.21.6
+      enhanced-resolve: 5.24.3
       jiti: 2.7.0
       lightningcss: 1.32.0
       magic-string: 0.30.21
       source-map-js: 1.2.1
-      tailwindcss: 4.3.1
+      tailwindcss: 4.3.3
 
-  '@tailwindcss/oxide-android-arm64@4.3.1':
+  '@tailwindcss/oxide-android-arm64@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-darwin-arm64@4.3.1':
+  '@tailwindcss/oxide-darwin-arm64@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-darwin-x64@4.3.1':
+  '@tailwindcss/oxide-darwin-x64@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-freebsd-x64@4.3.1':
+  '@tailwindcss/oxide-freebsd-x64@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.1':
+  '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-linux-arm64-gnu@4.3.1':
+  '@tailwindcss/oxide-linux-arm64-gnu@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-linux-arm64-musl@4.3.1':
+  '@tailwindcss/oxide-linux-arm64-musl@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-linux-x64-gnu@4.3.1':
+  '@tailwindcss/oxide-linux-x64-gnu@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-linux-x64-musl@4.3.1':
+  '@tailwindcss/oxide-linux-x64-musl@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-wasm32-wasi@4.3.1':
+  '@tailwindcss/oxide-wasm32-wasi@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-win32-arm64-msvc@4.3.1':
+  '@tailwindcss/oxide-win32-arm64-msvc@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide-win32-x64-msvc@4.3.1':
+  '@tailwindcss/oxide-win32-x64-msvc@4.3.3':
     optional: true
 
-  '@tailwindcss/oxide@4.3.1':
+  '@tailwindcss/oxide@4.3.3':
     optionalDependencies:
-      '@tailwindcss/oxide-android-arm64': 4.3.1
-      '@tailwindcss/oxide-darwin-arm64': 4.3.1
-      '@tailwindcss/oxide-darwin-x64': 4.3.1
-      '@tailwindcss/oxide-freebsd-x64': 4.3.1
-      '@tailwindcss/oxide-linux-arm-gnueabihf': 4.3.1
-      '@tailwindcss/oxide-linux-arm64-gnu': 4.3.1
-      '@tailwindcss/oxide-linux-arm64-musl': 4.3.1
-      '@tailwindcss/oxide-linux-x64-gnu': 4.3.1
-      '@tailwindcss/oxide-linux-x64-musl': 4.3.1
-      '@tailwindcss/oxide-wasm32-wasi': 4.3.1
-      '@tailwindcss/oxide-win32-arm64-msvc': 4.3.1
-      '@tailwindcss/oxide-win32-x64-msvc': 4.3.1
-
-  '@tailwindcss/vite@4.3.1(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))':
-    dependencies:
-      '@tailwindcss/node': 4.3.1
-      '@tailwindcss/oxide': 4.3.1
-      tailwindcss: 4.3.1
+      '@tailwindcss/oxide-android-arm64': 4.3.3
+      '@tailwindcss/oxide-darwin-arm64': 4.3.3
+      '@tailwindcss/oxide-darwin-x64': 4.3.3
+      '@tailwindcss/oxide-freebsd-x64': 4.3.3
+      '@tailwindcss/oxide-linux-arm-gnueabihf': 4.3.3
+      '@tailwindcss/oxide-linux-arm64-gnu': 4.3.3
+      '@tailwindcss/oxide-linux-arm64-musl': 4.3.3
+      '@tailwindcss/oxide-linux-x64-gnu': 4.3.3
+      '@tailwindcss/oxide-linux-x64-musl': 4.3.3
+      '@tailwindcss/oxide-wasm32-wasi': 4.3.3
+      '@tailwindcss/oxide-win32-arm64-msvc': 4.3.3
+      '@tailwindcss/oxide-win32-x64-msvc': 4.3.3
+
+  '@tailwindcss/vite@4.3.3(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))':
+    dependencies:
+      '@tailwindcss/node': 4.3.3
+      '@tailwindcss/oxide': 4.3.3
+      tailwindcss: 4.3.3
       vite: 5.4.21(@types/node@24.12.2)(lightningcss@1.32.0)
 
   '@types/esrecurse@4.3.1': {}
@@ -2756,103 +2782,103 @@ snapshots:
 
   '@types/web-bluetooth@0.0.21': {}
 
-  '@typescript-eslint/eslint-plugin@8.61.1(@typescript-eslint/parser@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)':
+  '@typescript-eslint/eslint-plugin@8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)':
     dependencies:
       '@eslint-community/regexpp': 4.12.2
-      '@typescript-eslint/parser': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
-      '@typescript-eslint/scope-manager': 8.61.1
-      '@typescript-eslint/type-utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
-      '@typescript-eslint/utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
-      '@typescript-eslint/visitor-keys': 8.61.1
-      eslint: 10.5.0(jiti@2.7.0)
-      ignore: 7.0.5
+      '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/scope-manager': 8.65.0
+      '@typescript-eslint/type-utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/visitor-keys': 8.65.0
+      eslint: 10.8.0(jiti@2.7.0)
+      ignore: 7.0.6
       natural-compare: 1.4.0
       ts-api-utils: 2.5.0(typescript@6.0.3)
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
 
-  '@typescript-eslint/parser@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)':
+  '@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)':
     dependencies:
-      '@typescript-eslint/scope-manager': 8.61.1
-      '@typescript-eslint/types': 8.61.1
-      '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3)
-      '@typescript-eslint/visitor-keys': 8.61.1
+      '@typescript-eslint/scope-manager': 8.65.0
+      '@typescript-eslint/types': 8.65.0
+      '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3)
+      '@typescript-eslint/visitor-keys': 8.65.0
       debug: 4.4.3
-      eslint: 10.5.0(jiti@2.7.0)
+      eslint: 10.8.0(jiti@2.7.0)
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
 
-  '@typescript-eslint/project-service@8.61.1(typescript@6.0.3)':
+  '@typescript-eslint/project-service@8.65.0(typescript@6.0.3)':
     dependencies:
-      '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3)
-      '@typescript-eslint/types': 8.61.1
+      '@typescript-eslint/tsconfig-utils': 8.65.0(typescript@6.0.3)
+      '@typescript-eslint/types': 8.65.0
       debug: 4.4.3
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
 
-  '@typescript-eslint/scope-manager@8.61.1':
+  '@typescript-eslint/scope-manager@8.65.0':
     dependencies:
-      '@typescript-eslint/types': 8.61.1
-      '@typescript-eslint/visitor-keys': 8.61.1
+      '@typescript-eslint/types': 8.65.0
+      '@typescript-eslint/visitor-keys': 8.65.0
 
-  '@typescript-eslint/tsconfig-utils@8.61.1(typescript@6.0.3)':
+  '@typescript-eslint/tsconfig-utils@8.65.0(typescript@6.0.3)':
     dependencies:
       typescript: 6.0.3
 
-  '@typescript-eslint/type-utils@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)':
+  '@typescript-eslint/type-utils@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)':
     dependencies:
-      '@typescript-eslint/types': 8.61.1
-      '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3)
-      '@typescript-eslint/utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/types': 8.65.0
+      '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3)
+      '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
       debug: 4.4.3
-      eslint: 10.5.0(jiti@2.7.0)
+      eslint: 10.8.0(jiti@2.7.0)
       ts-api-utils: 2.5.0(typescript@6.0.3)
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
 
-  '@typescript-eslint/types@8.61.1': {}
+  '@typescript-eslint/types@8.65.0': {}
 
-  '@typescript-eslint/typescript-estree@8.61.1(typescript@6.0.3)':
+  '@typescript-eslint/typescript-estree@8.65.0(typescript@6.0.3)':
     dependencies:
-      '@typescript-eslint/project-service': 8.61.1(typescript@6.0.3)
-      '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3)
-      '@typescript-eslint/types': 8.61.1
-      '@typescript-eslint/visitor-keys': 8.61.1
+      '@typescript-eslint/project-service': 8.65.0(typescript@6.0.3)
+      '@typescript-eslint/tsconfig-utils': 8.65.0(typescript@6.0.3)
+      '@typescript-eslint/types': 8.65.0
+      '@typescript-eslint/visitor-keys': 8.65.0
       debug: 4.4.3
-      minimatch: 10.2.5
-      semver: 7.8.4
+      minimatch: 10.2.6
+      semver: 7.8.5
       tinyglobby: 0.2.17
       ts-api-utils: 2.5.0(typescript@6.0.3)
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
 
-  '@typescript-eslint/utils@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)':
+  '@typescript-eslint/utils@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)':
     dependencies:
-      '@eslint-community/eslint-utils': 4.9.1(eslint@10.5.0(jiti@2.7.0))
-      '@typescript-eslint/scope-manager': 8.61.1
-      '@typescript-eslint/types': 8.61.1
-      '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3)
-      eslint: 10.5.0(jiti@2.7.0)
+      '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0))
+      '@typescript-eslint/scope-manager': 8.65.0
+      '@typescript-eslint/types': 8.65.0
+      '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3)
+      eslint: 10.8.0(jiti@2.7.0)
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
 
-  '@typescript-eslint/visitor-keys@8.61.1':
+  '@typescript-eslint/visitor-keys@8.65.0':
     dependencies:
-      '@typescript-eslint/types': 8.61.1
+      '@typescript-eslint/types': 8.65.0
       eslint-visitor-keys: 5.0.1
 
   '@ungap/structured-clone@1.3.0': {}
 
-  '@vitejs/plugin-vue@5.2.4(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))(vue@3.5.38(typescript@6.0.3))':
+  '@vitejs/plugin-vue@5.2.4(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))(vue@3.5.40(typescript@6.0.3))':
     dependencies:
       vite: 5.4.21(@types/node@24.12.2)(lightningcss@1.32.0)
-      vue: 3.5.38(typescript@6.0.3)
+      vue: 3.5.40(typescript@6.0.3)
 
   '@viz-js/viz@3.28.0': {}
 
@@ -2868,35 +2894,48 @@ snapshots:
       path-browserify: 1.0.1
       vscode-uri: 3.1.0
 
-  '@vue/compiler-core@3.5.38':
+  '@vue/compiler-core@3.5.40':
     dependencies:
       '@babel/parser': 7.29.7
-      '@vue/shared': 3.5.38
+      '@vue/shared': 3.5.40
+      entities: 7.0.1
+      estree-walker: 2.0.2
+      source-map-js: 1.2.1
+
+  '@vue/compiler-core@3.5.41':
+    dependencies:
+      '@babel/parser': 7.29.8
+      '@vue/shared': 3.5.41
       entities: 7.0.1
       estree-walker: 2.0.2
       source-map-js: 1.2.1
 
-  '@vue/compiler-dom@3.5.38':
+  '@vue/compiler-dom@3.5.40':
+    dependencies:
+      '@vue/compiler-core': 3.5.40
+      '@vue/shared': 3.5.40
+
+  '@vue/compiler-dom@3.5.41':
     dependencies:
-      '@vue/compiler-core': 3.5.38
-      '@vue/shared': 3.5.38
+      '@vue/compiler-core': 3.5.41
+      '@vue/shared': 3.5.41
 
-  '@vue/compiler-sfc@3.5.38':
+  '@vue/compiler-sfc@3.5.40':
     dependencies:
       '@babel/parser': 7.29.7
-      '@vue/compiler-core': 3.5.38
-      '@vue/compiler-dom': 3.5.38
-      '@vue/compiler-ssr': 3.5.38
-      '@vue/shared': 3.5.38
+      '@vue/compiler-core': 3.5.40
+      '@vue/compiler-dom': 3.5.40
+      '@vue/compiler-ssr': 3.5.40
+      '@vue/shared': 3.5.40
       estree-walker: 2.0.2
       magic-string: 0.30.21
-      postcss: 8.5.15
+      postcss: 8.5.22
       source-map-js: 1.2.1
 
-  '@vue/compiler-ssr@3.5.38':
+  '@vue/compiler-ssr@3.5.40':
     dependencies:
-      '@vue/compiler-dom': 3.5.38
-      '@vue/shared': 3.5.38
+      '@vue/compiler-dom': 3.5.40
+      '@vue/shared': 3.5.40
 
   '@vue/devtools-api@7.7.9':
     dependencies:
@@ -2916,48 +2955,50 @@ snapshots:
     dependencies:
       rfdc: 1.4.1
 
-  '@vue/language-core@3.3.5':
+  '@vue/language-core@3.3.9':
     dependencies:
       '@volar/language-core': 2.4.28
-      '@vue/compiler-dom': 3.5.38
-      '@vue/shared': 3.5.38
+      '@vue/compiler-dom': 3.5.41
+      '@vue/shared': 3.5.41
       alien-signals: 3.2.1
       muggle-string: 0.4.1
       path-browserify: 1.0.1
-      picomatch: 4.0.4
+      picomatch: 4.0.5
 
-  '@vue/reactivity@3.5.38':
+  '@vue/reactivity@3.5.40':
     dependencies:
-      '@vue/shared': 3.5.38
+      '@vue/shared': 3.5.40
 
-  '@vue/runtime-core@3.5.38':
+  '@vue/runtime-core@3.5.40':
     dependencies:
-      '@vue/reactivity': 3.5.38
-      '@vue/shared': 3.5.38
+      '@vue/reactivity': 3.5.40
+      '@vue/shared': 3.5.40
 
-  '@vue/runtime-dom@3.5.38':
+  '@vue/runtime-dom@3.5.40':
     dependencies:
-      '@vue/reactivity': 3.5.38
-      '@vue/runtime-core': 3.5.38
-      '@vue/shared': 3.5.38
+      '@vue/reactivity': 3.5.40
+      '@vue/runtime-core': 3.5.40
+      '@vue/shared': 3.5.40
       csstype: 3.2.3
 
-  '@vue/server-renderer@3.5.38(vue@3.5.38(typescript@6.0.3))':
+  '@vue/server-renderer@3.5.40':
     dependencies:
-      '@vue/compiler-ssr': 3.5.38
-      '@vue/shared': 3.5.38
-      vue: 3.5.38(typescript@6.0.3)
+      '@vue/compiler-ssr': 3.5.40
+      '@vue/runtime-dom': 3.5.40
+      '@vue/shared': 3.5.40
 
   '@vue/shared@3.5.32': {}
 
-  '@vue/shared@3.5.38': {}
+  '@vue/shared@3.5.40': {}
+
+  '@vue/shared@3.5.41': {}
 
   '@vueuse/core@12.8.2(typescript@6.0.3)':
     dependencies:
       '@types/web-bluetooth': 0.0.21
       '@vueuse/metadata': 12.8.2
       '@vueuse/shared': 12.8.2(typescript@6.0.3)
-      vue: 3.5.38(typescript@6.0.3)
+      vue: 3.5.40(typescript@6.0.3)
     transitivePeerDependencies:
       - typescript
 
@@ -2965,7 +3006,7 @@ snapshots:
     dependencies:
       '@vueuse/core': 12.8.2(typescript@6.0.3)
       '@vueuse/shared': 12.8.2(typescript@6.0.3)
-      vue: 3.5.38(typescript@6.0.3)
+      vue: 3.5.40(typescript@6.0.3)
     optionalDependencies:
       focus-trap: 7.8.0
     transitivePeerDependencies:
@@ -2975,15 +3016,15 @@ snapshots:
 
   '@vueuse/shared@12.8.2(typescript@6.0.3)':
     dependencies:
-      vue: 3.5.38(typescript@6.0.3)
+      vue: 3.5.40(typescript@6.0.3)
     transitivePeerDependencies:
       - typescript
 
-  acorn-jsx@5.3.2(acorn@8.17.0):
+  acorn-jsx@5.3.2(acorn@8.18.0):
     dependencies:
-      acorn: 8.17.0
+      acorn: 8.18.0
 
-  acorn@8.17.0: {}
+  acorn@8.18.0: {}
 
   ajv@6.15.0:
     dependencies:
@@ -3017,24 +3058,24 @@ snapshots:
 
   argparse@2.0.1: {}
 
-  autoprefixer@10.5.0(postcss@8.5.15):
+  autoprefixer@10.5.4(postcss@8.5.22):
     dependencies:
-      browserslist: 4.28.2
-      caniuse-lite: 1.0.30001788
+      browserslist: 4.28.7
+      caniuse-lite: 1.0.30001806
       fraction.js: 5.3.4
       picocolors: 1.1.1
-      postcss: 8.5.15
+      postcss: 8.5.22
       postcss-value-parser: 4.2.0
 
   balanced-match@4.0.4: {}
 
-  baseline-browser-mapping@2.10.20: {}
+  baseline-browser-mapping@2.11.1: {}
 
   birpc@2.9.0: {}
 
   boolbase@1.0.0: {}
 
-  brace-expansion@5.0.6:
+  brace-expansion@5.0.8:
     dependencies:
       balanced-match: 4.0.4
 
@@ -3042,15 +3083,15 @@ snapshots:
     dependencies:
       fill-range: 7.1.1
 
-  browserslist@4.28.2:
+  browserslist@4.28.7:
     dependencies:
-      baseline-browser-mapping: 2.10.20
-      caniuse-lite: 1.0.30001788
-      electron-to-chromium: 1.5.340
-      node-releases: 2.0.37
-      update-browserslist-db: 1.2.3(browserslist@4.28.2)
+      baseline-browser-mapping: 2.11.1
+      caniuse-lite: 1.0.30001806
+      electron-to-chromium: 1.5.395
+      node-releases: 2.0.51
+      update-browserslist-db: 1.2.3(browserslist@4.28.7)
 
-  caniuse-lite@1.0.30001788: {}
+  caniuse-lite@1.0.30001806: {}
 
   ccount@2.0.1: {}
 
@@ -3215,11 +3256,11 @@ snapshots:
       domelementtype: 2.3.0
       domhandler: 5.0.3
 
-  electron-to-chromium@1.5.340: {}
+  electron-to-chromium@1.5.395: {}
 
   emoji-regex-xs@1.0.0: {}
 
-  enhanced-resolve@5.21.6:
+  enhanced-resolve@5.24.3:
     dependencies:
       graceful-fs: 4.2.11
       tapable: 2.3.3
@@ -3258,22 +3299,22 @@ snapshots:
 
   escape-string-regexp@4.0.0: {}
 
-  eslint-config-prettier@10.1.8(eslint@10.5.0(jiti@2.7.0)):
+  eslint-config-prettier@10.1.8(eslint@10.8.0(jiti@2.7.0)):
     dependencies:
-      eslint: 10.5.0(jiti@2.7.0)
+      eslint: 10.8.0(jiti@2.7.0)
 
-  eslint-plugin-vue@10.9.2(@typescript-eslint/parser@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.5.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.5.0(jiti@2.7.0))):
+  eslint-plugin-vue@10.10.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0))):
     dependencies:
-      '@eslint-community/eslint-utils': 4.9.1(eslint@10.5.0(jiti@2.7.0))
-      eslint: 10.5.0(jiti@2.7.0)
+      '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0))
+      eslint: 10.8.0(jiti@2.7.0)
       natural-compare: 1.4.0
       nth-check: 2.1.1
-      postcss-selector-parser: 7.1.1
-      semver: 7.8.3
-      vue-eslint-parser: 10.4.1(eslint@10.5.0(jiti@2.7.0))
-      xml-name-validator: 4.0.0
+      postcss-selector-parser: 7.1.4
+      semver: 7.8.5
+      vue-eslint-parser: 10.4.1(eslint@10.8.0(jiti@2.7.0))
+      xml-name-validator: 5.0.0
     optionalDependencies:
-      '@typescript-eslint/parser': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
 
   eslint-scope@9.1.2:
     dependencies:
@@ -3286,12 +3327,12 @@ snapshots:
 
   eslint-visitor-keys@5.0.1: {}
 
-  eslint@10.5.0(jiti@2.7.0):
+  eslint@10.8.0(jiti@2.7.0):
     dependencies:
-      '@eslint-community/eslint-utils': 4.9.1(eslint@10.5.0(jiti@2.7.0))
+      '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0))
       '@eslint-community/regexpp': 4.12.2
       '@eslint/config-array': 0.23.5
-      '@eslint/config-helpers': 0.6.0
+      '@eslint/config-helpers': 0.7.0
       '@eslint/core': 1.2.1
       '@eslint/plugin-kit': 0.7.2
       '@humanfs/node': 0.16.8
@@ -3315,7 +3356,7 @@ snapshots:
       imurmurhash: 0.1.4
       is-glob: 4.0.3
       json-stable-stringify-without-jsonify: 1.0.1
-      minimatch: 10.2.5
+      minimatch: 10.2.6
       natural-compare: 1.4.0
       optionator: 0.9.4
     optionalDependencies:
@@ -3325,8 +3366,8 @@ snapshots:
 
   espree@11.2.0:
     dependencies:
-      acorn: 8.17.0
-      acorn-jsx: 5.3.2(acorn@8.17.0)
+      acorn: 8.18.0
+      acorn-jsx: 5.3.2(acorn@8.18.0)
       eslint-visitor-keys: 5.0.1
 
   esprima@4.0.1: {}
@@ -3367,9 +3408,9 @@ snapshots:
     dependencies:
       reusify: 1.1.0
 
-  fdir@6.5.0(picomatch@4.0.4):
+  fdir@6.5.0(picomatch@4.0.5):
     optionalDependencies:
-      picomatch: 4.0.4
+      picomatch: 4.0.5
 
   file-entry-cache@8.0.0:
     dependencies:
@@ -3386,10 +3427,10 @@ snapshots:
 
   flat-cache@4.0.1:
     dependencies:
-      flatted: 3.4.2
+      flatted: 3.4.3
       keyv: 4.5.4
 
-  flatted@3.4.2: {}
+  flatted@3.4.3: {}
 
   focus-trap@7.8.0:
     dependencies:
@@ -3445,7 +3486,7 @@ snapshots:
 
   ignore@5.3.2: {}
 
-  ignore@7.0.5: {}
+  ignore@7.0.6: {}
 
   imurmurhash@0.1.4: {}
 
@@ -3546,7 +3587,7 @@ snapshots:
       lightningcss-win32-arm64-msvc: 1.32.0
       lightningcss-win32-x64-msvc: 1.32.0
 
-  linkify-it@5.0.1:
+  linkify-it@5.0.2:
     dependencies:
       uc.micro: 2.1.0
 
@@ -3564,11 +3605,11 @@ snapshots:
 
   markdown-it-footnote@4.0.0: {}
 
-  markdown-it@14.2.0:
+  markdown-it@14.3.0:
     dependencies:
       argparse: 2.0.1
       entities: 4.5.0
-      linkify-it: 5.0.1
+      linkify-it: 5.0.2
       mdurl: 2.0.0
       punycode.js: 2.3.1
       uc.micro: 2.1.0
@@ -3615,9 +3656,9 @@ snapshots:
       braces: 3.0.3
       picomatch: 2.3.2
 
-  minimatch@10.2.5:
+  minimatch@10.2.6:
     dependencies:
-      brace-expansion: 5.0.6
+      brace-expansion: 5.0.8
 
   minisearch@7.2.0: {}
 
@@ -3631,6 +3672,8 @@ snapshots:
 
   nanoid@3.3.12: {}
 
+  nanoid@3.3.16: {}
+
   natural-compare@1.4.0: {}
 
   nearley@2.20.1:
@@ -3640,7 +3683,7 @@ snapshots:
       railroad-diagrams: 1.0.0
       randexp: 0.4.6
 
-  node-releases@2.0.37: {}
+  node-releases@2.0.51: {}
 
   nth-check@2.1.1:
     dependencies:
@@ -3681,9 +3724,9 @@ snapshots:
 
   picomatch@2.3.2: {}
 
-  picomatch@4.0.4: {}
+  picomatch@4.0.5: {}
 
-  postcss-selector-parser@7.1.1:
+  postcss-selector-parser@7.1.4:
     dependencies:
       cssesc: 3.0.0
       util-deprecate: 1.0.2
@@ -3696,9 +3739,9 @@ snapshots:
       picocolors: 1.1.1
       source-map-js: 1.2.1
 
-  postcss@8.5.15:
+  postcss@8.5.22:
     dependencies:
-      nanoid: 3.3.12
+      nanoid: 3.3.16
       picocolors: 1.1.1
       source-map-js: 1.2.1
 
@@ -3706,17 +3749,17 @@ snapshots:
 
   prelude-ls@1.2.1: {}
 
-  prettier-plugin-tailwindcss@0.8.0(@ianvs/prettier-plugin-sort-imports@4.7.1(@vue/compiler-sfc@3.5.38)(prettier@3.8.4))(prettier@3.8.4):
+  prettier-plugin-tailwindcss@0.8.1(@ianvs/prettier-plugin-sort-imports@4.7.1(@vue/compiler-sfc@3.5.40)(prettier@3.9.6))(prettier@3.9.6):
     dependencies:
-      prettier: 3.8.4
+      prettier: 3.9.6
     optionalDependencies:
-      '@ianvs/prettier-plugin-sort-imports': 4.7.1(@vue/compiler-sfc@3.5.38)(prettier@3.8.4)
+      '@ianvs/prettier-plugin-sort-imports': 4.7.1(@vue/compiler-sfc@3.5.40)(prettier@3.9.6)
 
-  prettier-plugin-vitepress@0.0.10(prettier@3.8.4):
+  prettier-plugin-vitepress@0.0.10(prettier@3.9.6):
     dependencies:
-      prettier: 3.8.4
+      prettier: 3.9.6
 
-  prettier@3.8.4: {}
+  prettier@3.9.6: {}
 
   property-information@7.1.0: {}
 
@@ -3803,9 +3846,7 @@ snapshots:
 
   semver@7.8.1: {}
 
-  semver@7.8.3: {}
-
-  semver@7.8.4: {}
+  semver@7.8.5: {}
 
   shebang-command@2.0.0:
     dependencies:
@@ -3832,7 +3873,7 @@ snapshots:
 
   sprintf-js@1.0.3: {}
 
-  sql-formatter@15.8.1:
+  sql-formatter@15.8.2:
     dependencies:
       argparse: 2.0.1
       nearley: 2.20.1
@@ -3848,7 +3889,7 @@ snapshots:
     dependencies:
       copy-anything: 4.0.5
 
-  svgo@4.0.1:
+  svgo@4.0.2:
     dependencies:
       commander: 11.1.0
       css-select: 5.2.2
@@ -3860,14 +3901,14 @@ snapshots:
 
   tabbable@6.4.0: {}
 
-  tailwindcss@4.3.1: {}
+  tailwindcss@4.3.3: {}
 
   tapable@2.3.3: {}
 
   tinyglobby@0.2.17:
     dependencies:
-      fdir: 6.5.0(picomatch@4.0.4)
-      picomatch: 4.0.4
+      fdir: 6.5.0(picomatch@4.0.5)
+      picomatch: 4.0.5
 
   to-regex-range@5.0.1:
     dependencies:
@@ -3887,13 +3928,13 @@ snapshots:
     dependencies:
       prelude-ls: 1.2.1
 
-  typescript-eslint@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3):
+  typescript-eslint@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3):
     dependencies:
-      '@typescript-eslint/eslint-plugin': 8.61.1(@typescript-eslint/parser@8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
-      '@typescript-eslint/parser': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
-      '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3)
-      '@typescript-eslint/utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3)
-      eslint: 10.5.0(jiti@2.7.0)
+      '@typescript-eslint/eslint-plugin': 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
+      '@typescript-eslint/typescript-estree': 8.65.0(typescript@6.0.3)
+      '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@2.7.0))(typescript@6.0.3)
+      eslint: 10.8.0(jiti@2.7.0)
       typescript: 6.0.3
     transitivePeerDependencies:
       - supports-color
@@ -3927,9 +3968,9 @@ snapshots:
       unist-util-is: 6.0.1
       unist-util-visit-parents: 6.0.2
 
-  update-browserslist-db@1.2.3(browserslist@4.28.2):
+  update-browserslist-db@1.2.3(browserslist@4.28.7):
     dependencies:
-      browserslist: 4.28.2
+      browserslist: 4.28.7
       escalade: 3.2.0
       picocolors: 1.1.1
 
@@ -3941,229 +3982,229 @@ snapshots:
 
   vega-canvas@2.0.0: {}
 
-  vega-crossfilter@5.1.0:
+  vega-crossfilter@5.1.2:
     dependencies:
       d3-array: 3.2.4
-      vega-dataflow: 6.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-util: 2.1.2
 
-  vega-dataflow@6.1.0:
+  vega-dataflow@6.1.2:
     dependencies:
-      vega-format: 2.1.0
-      vega-loader: 5.1.0
-      vega-util: 2.1.0
+      vega-format: 2.1.2
+      vega-loader: 5.1.2
+      vega-util: 2.1.2
 
-  vega-encode@5.1.0:
+  vega-encode@5.2.1:
     dependencies:
       d3-array: 3.2.4
       d3-interpolate: 3.0.1
-      vega-dataflow: 6.1.0
-      vega-scale: 8.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-scale: 8.1.2
+      vega-util: 2.1.2
 
   vega-event-selector@4.0.0: {}
 
-  vega-expression@6.1.0:
+  vega-expression@6.2.1:
     dependencies:
       '@types/estree': 1.0.9
-      vega-util: 2.1.0
+      vega-util: 2.1.2
 
-  vega-force@5.1.0:
+  vega-force@5.1.2:
     dependencies:
       d3-force: 3.0.0
-      vega-dataflow: 6.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-util: 2.1.2
 
-  vega-format@2.1.0:
+  vega-format@2.1.2:
     dependencies:
       d3-array: 3.2.4
       d3-format: 3.1.2
       d3-time-format: 4.1.0
-      vega-time: 3.1.0
-      vega-util: 2.1.0
+      vega-time: 3.2.1
+      vega-util: 2.1.2
 
-  vega-functions@6.1.1:
+  vega-functions@6.1.3:
     dependencies:
       d3-array: 3.2.4
       d3-color: 3.1.0
       d3-geo: 3.1.1
-      vega-dataflow: 6.1.0
-      vega-expression: 6.1.0
-      vega-scale: 8.1.0
-      vega-scenegraph: 5.1.0
-      vega-selections: 6.1.2
+      vega-dataflow: 6.1.2
+      vega-expression: 6.2.1
+      vega-scale: 8.1.2
+      vega-scenegraph: 5.2.1
+      vega-selections: 6.1.4
       vega-statistics: 2.0.0
-      vega-time: 3.1.0
-      vega-util: 2.1.0
+      vega-time: 3.2.1
+      vega-util: 2.1.2
 
-  vega-geo@5.1.0:
+  vega-geo@5.1.2:
     dependencies:
       d3-array: 3.2.4
       d3-color: 3.1.0
       d3-geo: 3.1.1
       vega-canvas: 2.0.0
-      vega-dataflow: 6.1.0
-      vega-projection: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-projection: 2.1.2
       vega-statistics: 2.0.0
-      vega-util: 2.1.0
+      vega-util: 2.1.2
 
-  vega-hierarchy@5.1.0:
+  vega-hierarchy@5.1.2:
     dependencies:
       d3-hierarchy: 3.1.2
-      vega-dataflow: 6.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-util: 2.1.2
 
-  vega-label@2.1.0:
+  vega-label@2.1.2:
     dependencies:
       vega-canvas: 2.0.0
-      vega-dataflow: 6.1.0
-      vega-scenegraph: 5.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-scenegraph: 5.2.1
+      vega-util: 2.1.2
 
-  vega-loader@5.1.0:
+  vega-loader@5.1.2:
     dependencies:
       d3-dsv: 3.0.1
       topojson-client: 3.1.0
-      vega-format: 2.1.0
-      vega-util: 2.1.0
+      vega-format: 2.1.2
+      vega-util: 2.1.2
 
-  vega-parser@7.1.0:
+  vega-parser@7.1.2:
     dependencies:
-      vega-dataflow: 6.1.0
+      vega-dataflow: 6.1.2
       vega-event-selector: 4.0.0
-      vega-functions: 6.1.1
-      vega-scale: 8.1.0
-      vega-util: 2.1.0
+      vega-functions: 6.1.3
+      vega-scale: 8.1.2
+      vega-util: 2.1.2
 
-  vega-projection@2.1.0:
+  vega-projection@2.1.2:
     dependencies:
       d3-geo: 3.1.1
       d3-geo-projection: 4.0.0
-      vega-scale: 8.1.0
+      vega-scale: 8.1.2
 
-  vega-regression@2.1.0:
+  vega-regression@2.1.2:
     dependencies:
       d3-array: 3.2.4
-      vega-dataflow: 6.1.0
+      vega-dataflow: 6.1.2
       vega-statistics: 2.0.0
-      vega-util: 2.1.0
+      vega-util: 2.1.2
 
-  vega-runtime@7.1.0:
+  vega-runtime@7.1.2:
     dependencies:
-      vega-dataflow: 6.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-util: 2.1.2
 
-  vega-scale@8.1.0:
+  vega-scale@8.1.2:
     dependencies:
       d3-array: 3.2.4
       d3-interpolate: 3.0.1
       d3-scale: 4.0.2
       d3-scale-chromatic: 3.1.0
-      vega-time: 3.1.0
-      vega-util: 2.1.0
+      vega-time: 3.2.1
+      vega-util: 2.1.2
 
-  vega-scenegraph@5.1.0:
+  vega-scenegraph@5.2.1:
     dependencies:
       d3-path: 3.1.0
       d3-shape: 3.2.0
       vega-canvas: 2.0.0
-      vega-loader: 5.1.0
-      vega-scale: 8.1.0
-      vega-util: 2.1.0
+      vega-loader: 5.1.2
+      vega-scale: 8.1.2
+      vega-util: 2.1.2
 
-  vega-selections@6.1.2:
+  vega-selections@6.1.4:
     dependencies:
       d3-array: 3.2.4
-      vega-expression: 6.1.0
-      vega-util: 2.1.0
+      vega-expression: 6.2.1
+      vega-util: 2.1.2
 
   vega-statistics@2.0.0:
     dependencies:
       d3-array: 3.2.4
 
-  vega-time@3.1.0:
+  vega-time@3.2.1:
     dependencies:
       d3-array: 3.2.4
       d3-time: 3.1.0
-      vega-util: 2.1.0
+      vega-util: 2.1.2
 
-  vega-transforms@5.1.0:
+  vega-transforms@5.2.1:
     dependencies:
       d3-array: 3.2.4
-      vega-dataflow: 6.1.0
+      vega-dataflow: 6.1.2
       vega-statistics: 2.0.0
-      vega-time: 3.1.0
-      vega-util: 2.1.0
+      vega-time: 3.2.1
+      vega-util: 2.1.2
 
-  vega-typings@2.1.0:
+  vega-typings@2.2.0:
     dependencies:
       '@types/geojson': 7946.0.16
       vega-event-selector: 4.0.0
-      vega-expression: 6.1.0
-      vega-util: 2.1.0
+      vega-expression: 6.2.1
+      vega-util: 2.1.2
 
-  vega-util@2.1.0: {}
+  vega-util@2.1.2: {}
 
-  vega-view-transforms@5.1.0:
+  vega-view-transforms@5.2.1:
     dependencies:
-      vega-dataflow: 6.1.0
-      vega-scenegraph: 5.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-scenegraph: 5.2.1
+      vega-util: 2.1.2
 
-  vega-view@6.1.0:
+  vega-view@6.1.2:
     dependencies:
       d3-array: 3.2.4
       d3-timer: 3.0.1
-      vega-dataflow: 6.1.0
-      vega-format: 2.1.0
-      vega-functions: 6.1.1
-      vega-runtime: 7.1.0
-      vega-scenegraph: 5.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-format: 2.1.2
+      vega-functions: 6.1.3
+      vega-runtime: 7.1.2
+      vega-scenegraph: 5.2.1
+      vega-util: 2.1.2
 
-  vega-voronoi@5.1.0:
+  vega-voronoi@5.1.2:
     dependencies:
       d3-delaunay: 6.0.4
-      vega-dataflow: 6.1.0
-      vega-util: 2.1.0
+      vega-dataflow: 6.1.2
+      vega-util: 2.1.2
 
-  vega-wordcloud@5.1.0:
+  vega-wordcloud@5.1.2:
     dependencies:
       vega-canvas: 2.0.0
-      vega-dataflow: 6.1.0
-      vega-scale: 8.1.0
+      vega-dataflow: 6.1.2
+      vega-scale: 8.1.2
       vega-statistics: 2.0.0
-      vega-util: 2.1.0
+      vega-util: 2.1.2
 
-  vega@6.2.0:
+  vega@6.3.1:
     dependencies:
-      vega-crossfilter: 5.1.0
-      vega-dataflow: 6.1.0
-      vega-encode: 5.1.0
+      vega-crossfilter: 5.1.2
+      vega-dataflow: 6.1.2
+      vega-encode: 5.2.1
       vega-event-selector: 4.0.0
-      vega-expression: 6.1.0
-      vega-force: 5.1.0
-      vega-format: 2.1.0
-      vega-functions: 6.1.1
-      vega-geo: 5.1.0
-      vega-hierarchy: 5.1.0
-      vega-label: 2.1.0
-      vega-loader: 5.1.0
-      vega-parser: 7.1.0
-      vega-projection: 2.1.0
-      vega-regression: 2.1.0
-      vega-runtime: 7.1.0
-      vega-scale: 8.1.0
-      vega-scenegraph: 5.1.0
+      vega-expression: 6.2.1
+      vega-force: 5.1.2
+      vega-format: 2.1.2
+      vega-functions: 6.1.3
+      vega-geo: 5.1.2
+      vega-hierarchy: 5.1.2
+      vega-label: 2.1.2
+      vega-loader: 5.1.2
+      vega-parser: 7.1.2
+      vega-projection: 2.1.2
+      vega-regression: 2.1.2
+      vega-runtime: 7.1.2
+      vega-scale: 8.1.2
+      vega-scenegraph: 5.2.1
       vega-statistics: 2.0.0
-      vega-time: 3.1.0
-      vega-transforms: 5.1.0
-      vega-typings: 2.1.0
-      vega-util: 2.1.0
-      vega-view: 6.1.0
-      vega-view-transforms: 5.1.0
-      vega-voronoi: 5.1.0
-      vega-wordcloud: 5.1.0
+      vega-time: 3.2.1
+      vega-transforms: 5.2.1
+      vega-typings: 2.2.0
+      vega-util: 2.1.2
+      vega-view: 6.1.2
+      vega-view-transforms: 5.2.1
+      vega-voronoi: 5.1.2
+      vega-wordcloud: 5.1.2
 
   vfile-message@4.0.3:
     dependencies:
@@ -4185,7 +4226,7 @@ snapshots:
       fsevents: 2.3.3
       lightningcss: 1.32.0
 
-  vitepress@1.6.4(@algolia/client-search@5.50.2)(@types/node@24.12.2)(lightningcss@1.32.0)(postcss@8.5.15)(search-insights@2.17.3)(typescript@6.0.3):
+  vitepress@1.6.4(@algolia/client-search@5.50.2)(@types/node@24.12.2)(lightningcss@1.32.0)(postcss@8.5.22)(search-insights@2.17.3)(typescript@6.0.3):
     dependencies:
       '@docsearch/css': 3.8.2
       '@docsearch/js': 3.8.2(@algolia/client-search@5.50.2)(search-insights@2.17.3)
@@ -4194,7 +4235,7 @@ snapshots:
       '@shikijs/transformers': 2.5.0
       '@shikijs/types': 2.5.0
       '@types/markdown-it': 14.1.2
-      '@vitejs/plugin-vue': 5.2.4(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))(vue@3.5.38(typescript@6.0.3))
+      '@vitejs/plugin-vue': 5.2.4(vite@5.4.21(@types/node@24.12.2)(lightningcss@1.32.0))(vue@3.5.40(typescript@6.0.3))
       '@vue/devtools-api': 7.7.9
       '@vue/shared': 3.5.32
       '@vueuse/core': 12.8.2(typescript@6.0.3)
@@ -4204,9 +4245,9 @@ snapshots:
       minisearch: 7.2.0
       shiki: 2.5.0
       vite: 5.4.21(@types/node@24.12.2)(lightningcss@1.32.0)
-      vue: 3.5.38(typescript@6.0.3)
+      vue: 3.5.40(typescript@6.0.3)
     optionalDependencies:
-      postcss: 8.5.15
+      postcss: 8.5.22
     transitivePeerDependencies:
       - '@algolia/client-search'
       - '@types/node'
@@ -4236,10 +4277,10 @@ snapshots:
 
   vscode-uri@3.1.0: {}
 
-  vue-eslint-parser@10.4.1(eslint@10.5.0(jiti@2.7.0)):
+  vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)):
     dependencies:
       debug: 4.4.3
-      eslint: 10.5.0(jiti@2.7.0)
+      eslint: 10.8.0(jiti@2.7.0)
       eslint-scope: 9.1.2
       eslint-visitor-keys: 5.0.1
       espree: 11.2.0
@@ -4248,19 +4289,19 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  vue-tsc@3.3.5(typescript@6.0.3):
+  vue-tsc@3.3.9(typescript@6.0.3):
     dependencies:
       '@volar/typescript': 2.4.28
-      '@vue/language-core': 3.3.5
+      '@vue/language-core': 3.3.9
       typescript: 6.0.3
 
-  vue@3.5.38(typescript@6.0.3):
+  vue@3.5.40(typescript@6.0.3):
     dependencies:
-      '@vue/compiler-dom': 3.5.38
-      '@vue/compiler-sfc': 3.5.38
-      '@vue/runtime-dom': 3.5.38
-      '@vue/server-renderer': 3.5.38(vue@3.5.38(typescript@6.0.3))
-      '@vue/shared': 3.5.38
+      '@vue/compiler-dom': 3.5.40
+      '@vue/compiler-sfc': 3.5.40
+      '@vue/runtime-dom': 3.5.40
+      '@vue/server-renderer': 3.5.40
+      '@vue/shared': 3.5.40
     optionalDependencies:
       typescript: 6.0.3
 
@@ -4270,7 +4311,7 @@ snapshots:
 
   word-wrap@1.2.5: {}
 
-  xml-name-validator@4.0.0: {}
+  xml-name-validator@5.0.0: {}
 
   yocto-queue@0.1.0: {}
 
diff --git a/pyproject.toml b/pyproject.toml
index 27608230b..f8c28b302 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "pysail"
-version = "0.6.4"
+version = "0.7.0"
 description = "Sail Python library"
 authors = [
     { name = "LakeSail", email = "hello@lakesail.com" },
@@ -32,7 +32,7 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     # The dependencies for testing the installed package.
-    "pyspark-client==4.1.1",
+    "pyspark-client==4.2.0",
     "pandas<3",
     "duckdb>=1.0,<2",
     "pytest>=9.1,<10",
@@ -93,7 +93,7 @@ python = "3.11"
 installer = "pip"
 skip-install = true
 dependencies = [
-    "pyspark[connect]==4.1.1",
+    "pyspark[connect]==4.2.0",
     "ibis-framework>=12,<13",
     "pandas<3",
     "pytest>=9.1,<10",
@@ -173,18 +173,20 @@ dependencies = [
 ]
 
 [[tool.hatch.envs.test.matrix]]
-spark = ["3.5.7", "4.0.1", "4.1.1"]
+spark = ["3.5.9", "4.0.4", "4.1.3", "4.2.0"]
 
 [tool.hatch.envs.test.overrides]
 matrix.spark.path = [
-    { value = ".venvs/test.spark-3.5.7", if = ["3.5.7"] },
-    { value = ".venvs/test.spark-4.0.1", if = ["4.0.1"] },
-    { value = ".venvs/test.spark-4.1.1", if = ["4.1.1"] },
+    { value = ".venvs/test.spark-3.5.9", if = ["3.5.9"] },
+    { value = ".venvs/test.spark-4.0.4", if = ["4.0.4"] },
+    { value = ".venvs/test.spark-4.1.3", if = ["4.1.3"] },
+    { value = ".venvs/test.spark-4.2.0", if = ["4.2.0"] },
 ]
 matrix.spark.extra-dependencies = [
-    { value = "pyspark[connect]==3.5.7", if = ["3.5.7"] },
-    { value = "pyspark[connect]==4.0.1", if = ["4.0.1"] },
-    { value = "pyspark[connect]==4.1.1", if = ["4.1.1"] },
+    { value = "pyspark[connect]==3.5.9", if = ["3.5.9"] },
+    { value = "pyspark[connect]==4.0.4", if = ["4.0.4"] },
+    { value = "pyspark[connect]==4.1.3", if = ["4.1.3"] },
+    { value = "pyspark[connect]==4.2.0", if = ["4.2.0"] },
 ]
 env.CI.installer = "uv"
 
@@ -201,15 +203,15 @@ dependencies = [
 ]
 
 [[tool.hatch.envs.test-spark.matrix]]
-spark = ["3.5.7", "4.1.1"]
+spark = ["3.5.9", "4.2.0"]
 
 [tool.hatch.envs.test-spark.extra-scripts]
 install-pyspark = "\"{env:HATCH_UV}\" pip install --force-reinstall 'pyspark[connect] @ opt/spark/python/dist/pyspark-{matrix:spark}.tar.gz' 'pandas<3'"
 
 [tool.hatch.envs.test-spark.overrides]
 matrix.spark.path = [
-    { value = ".venvs/test-spark.spark-3.5.7", if = ["3.5.7"] },
-    { value = ".venvs/test-spark.spark-4.1.1", if = ["4.1.1"] },
+    { value = ".venvs/test-spark.spark-3.5.9", if = ["3.5.9"] },
+    { value = ".venvs/test-spark.spark-4.2.0", if = ["4.2.0"] },
 ]
 env.CI.installer = "uv"
 
@@ -218,7 +220,7 @@ python = "3.11"
 installer = "pip"
 skip-install = true
 dependencies = [
-    "pyspark[connect]==3.5.7",
+    "pyspark[connect]==3.5.9",
     "ibis-framework[pyspark]>=12,<13",
     "sqlglot>=23.4,!=26.32.0",
     # The test dependencies are borrowed from `pyproject.toml` of the Ibis project.
diff --git a/python/pysail/__init__.py b/python/pysail/__init__.py
index 903a8704c..7dcb7c4f2 100644
--- a/python/pysail/__init__.py
+++ b/python/pysail/__init__.py
@@ -2,6 +2,6 @@
 # We have a CI step that verifies this.
 # We cannot use the Hatch dynamic version feature since the project
 # may be built with Maturin outside of Hatch.
-__version__: str = "0.6.4"
+__version__: str = "0.7.0"
 
 __all__ = ["__version__"]
diff --git a/python/pysail/data/compatibility/catalog.json b/python/pysail/data/compatibility/catalog.json
index 2e2aa3a07..a8fece505 100644
--- a/python/pysail/data/compatibility/catalog.json
+++ b/python/pysail/data/compatibility/catalog.json
@@ -87,7 +87,8 @@
   {
     "module": "pyspark.sql.Catalog",
     "function": "listFunctions",
-    "status": "planned"
+    "status": "partially supported",
+    "note": "Persistent catalog functions are not supported yet."
   },
   {
     "module": "pyspark.sql.Catalog",
diff --git a/python/pysail/data/compatibility/dataframe.json b/python/pysail/data/compatibility/dataframe.json
index b67b134ec..e0a6fc22e 100644
--- a/python/pysail/data/compatibility/dataframe.json
+++ b/python/pysail/data/compatibility/dataframe.json
@@ -23,8 +23,7 @@
   {
     "module": "pyspark.sql.DataFrame",
     "function": "checkpoint",
-    "status": "planned",
-    "ref": "https://github.com/lakehq/sail/issues/476"
+    "status": "supported"
   },
   {
     "module": "pyspark.sql.DataFrame",
@@ -205,8 +204,7 @@
   {
     "module": "pyspark.sql.DataFrame",
     "function": "inputFiles",
-    "status": "planned",
-    "ref": "https://github.com/lakehq/sail/issues/481"
+    "status": "supported"
   },
   {
     "module": "pyspark.sql.DataFrame",
@@ -558,4 +556,4 @@
     "function": "writeTo",
     "status": "supported"
   }
-]
\ No newline at end of file
+]
diff --git a/python/pysail/data/compatibility/functions/scalar/xml.json b/python/pysail/data/compatibility/functions/scalar/xml.json
index 7c85b2d18..5e7560491 100644
--- a/python/pysail/data/compatibility/functions/scalar/xml.json
+++ b/python/pysail/data/compatibility/functions/scalar/xml.json
@@ -2,8 +2,7 @@
   {
     "module": "pyspark.sql.functions",
     "function": "from_xml",
-    "status": "planned",
-    "ref": "https://github.com/lakehq/sail/issues/231"
+    "status": "supported"
   },
   {
     "module": "pyspark.sql.functions",
diff --git a/python/pysail/examples/spark/onelake_catalog.py b/python/pysail/examples/spark/onelake_catalog.py
index 63c2324c4..fb5d60b47 100644
--- a/python/pysail/examples/spark/onelake_catalog.py
+++ b/python/pysail/examples/spark/onelake_catalog.py
@@ -33,7 +33,7 @@ def main():
 
     # Set up environment with the token
     os.environ["AZURE_STORAGE_TOKEN"] = token
-    os.environ["SAIL_CATALOG__LIST"] = '[{type="onelake", name="onelake", url="duckrun/data.Lakehouse"}]'
+    os.environ["SAIL_CATALOG__LIST"] = '[{type="onelake", name="onelake", url="duckrun/data.Lakehouse", api="delta"}]'
     os.environ["SAIL_CATALOG__DEFAULT_CATALOG"] = "onelake"
 
     # Start Sail server
diff --git a/python/pysail/testing/spark/session.py b/python/pysail/testing/spark/session.py
index 71be07ac3..3240c3c26 100644
--- a/python/pysail/testing/spark/session.py
+++ b/python/pysail/testing/spark/session.py
@@ -25,7 +25,12 @@ def __init__(self, remote: str) -> None:
 
     def create(self) -> SparkSession:
         app = f"test-{uuid.uuid4().hex}"
-        session = SparkSession.builder.appName(app).remote(self._remote).create()
+        builder = SparkSession.builder.appName(app).remote(self._remote)
+        # PySpark 4.1 removed `local` connection-string support from `.create()`
+        # (raises UNSUPPORTED_LOCAL_CONNECTION_STRING), but `.getOrCreate()` still
+        # boots a local JVM-backed Spark Connect server. Use it so feature tests can
+        # run against real Spark via `SPARK_REMOTE="local"`.
+        session = builder.getOrCreate() if self._remote.startswith("local") else builder.create()
         configure_spark_session(session)
         patch_spark_connect_session(session)
         self._sessions.append(session)
diff --git a/python/pysail/testing/spark/steps/delta.py b/python/pysail/testing/spark/steps/delta.py
index f67df158c..8d516e635 100644
--- a/python/pysail/testing/spark/steps/delta.py
+++ b/python/pysail/testing/spark/steps/delta.py
@@ -236,14 +236,21 @@ def _normalize_column_mapping_schema(schema: object, field_path: str = "") -> ob
     normalized = dict(schema)
     if "fields" in normalized and isinstance(normalized["fields"], list):
         for i, field in enumerate(normalized["fields"]):
+            if not isinstance(field, dict):
+                continue
             current_path = f"{field_path}.{i}" if field_path else str(i)
-            if isinstance(field, dict) and "metadata" in field:
-                meta = field.get("metadata", {})
+            meta = field.get("metadata")
+            if isinstance(meta, dict):
                 phys = meta.get("delta.columnMapping.physicalName", "")
                 if isinstance(phys, str) and phys.startswith("col-"):
                     meta["delta.columnMapping.physicalName"] = f""
-            if isinstance(field, dict) and "type" in field and isinstance(field["type"], dict):
-                field["type"] = _normalize_column_mapping_schema(field["type"], field_path=current_path)
+            field_type = field.get("type")
+            if isinstance(field_type, dict):
+                field["type"] = _normalize_column_mapping_schema(field_type, field_path=current_path)
+    for child_key in ("elementType", "keyType", "valueType"):
+        child_type = normalized.get(child_key)
+        if isinstance(child_type, dict):
+            normalized[child_key] = _normalize_column_mapping_schema(child_type, field_path=field_path)
     return normalized
 
 
@@ -283,6 +290,21 @@ def delta_log_cache() -> dict[str, dict]:
     return {}
 
 
+@given(parsers.parse("append query to delta table in {location_var} with mergeSchema"))
+def append_query_to_delta_table_with_merge_schema(
+    docstring: str,
+    location_var: str,
+    variables: dict,
+    spark,
+) -> None:
+    location = variables.get(location_var)
+    assert location is not None, f"Variable {location_var!r} not found"
+    spark.sql(docstring).write.format("delta").mode("append").option(
+        "mergeSchema",
+        "true",
+    ).save(str(location.path.absolute()))
+
+
 def _delta_log_compute(which: str, variables: dict, delta_log_cache: dict[str, dict]) -> dict:
     cached = delta_log_cache.get(which)
     if isinstance(cached, dict):
diff --git a/python/pysail/testing/spark/steps/plan.py b/python/pysail/testing/spark/steps/plan.py
index 059e48f00..4eabca51e 100644
--- a/python/pysail/testing/spark/steps/plan.py
+++ b/python/pysail/testing/spark/steps/plan.py
@@ -36,7 +36,7 @@ def _normalize_metrics_block(match: re.Match[str]) -> str:
 
     # Normalize temp paths / file URIs that appear in plans.
     pytest_tmp_prefix = re.compile(
-        r"(^|[\s\[\(=,:{\"])"
+        r"(^|[\s\[\(=,:{\"]|[A-Za-z][A-Za-z0-9+.\-]+://)"
         r"(?!\[)"
         r"(?:(?:[A-Za-z]:)?/|private/|tmp/)"
         r"(?:[^ \t\r\n\),\]/]+/)*"
@@ -68,12 +68,31 @@ def normalize_path(path: str) -> str:
         lambda m: f"table_path=file://{normalize_path(m.group(1))}",
         text,
     )
+    text = re.sub(
+        r"data_file=file://([^\s\),]+)",
+        lambda m: f"data_file=file://{normalize_path(m.group(1))}",
+        text,
+    )
     text = re.sub(
         r'location: "([^"]+)"',
         lambda m: f'location: "{normalize_path(m.group(1))}"',
         text,
     )
     text = pytest_tmp_prefix.sub(lambda m: f"{m.group(1)}/", text)
+    # Preserve checkpoint partition layout while removing generated identifiers.
+    checkpoint_uuid = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+    text = re.sub(
+        rf"(__checkpoint_testing__/){checkpoint_uuid}/{checkpoint_uuid}",
+        r"\1/",
+        text,
+        flags=re.IGNORECASE,
+    )
+    text = re.sub(
+        rf"(__checkpoint_testing__///part-\d{{5}}-){checkpoint_uuid}(\.parquet)",
+        r"\1\2",
+        text,
+        flags=re.IGNORECASE,
+    )
     # Normalize Delta Lake parquet files: part---c.snappy.parquet
     text = re.sub(
         r"part-\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-c\d+\.snappy\.parquet",
@@ -135,6 +154,13 @@ def _normalize_file_groups_block(match: re.Match[str]) -> str:
         end = block.rfind("]")
         if start == -1 or end == -1 or end <= start:
             return block
+
+        # Checkpoint scans retain the input's physical partition layout. For iterative queries,
+        # repartitioning can leave different partitions empty across runs, so individual checkpoint
+        # files are execution details rather than stable plan properties.
+        if "__checkpoint_testing__/" in block:
+            return block[:start] + "" + block[end + 1 :]
+
         groups_list = block[start : end + 1]
 
         # Parse top-level groups inside the outer list.
diff --git a/python/pysail/testing/spark/steps/sql.py b/python/pysail/testing/spark/steps/sql.py
index 28782dff2..ac6d6c7c0 100644
--- a/python/pysail/testing/spark/steps/sql.py
+++ b/python/pysail/testing/spark/steps/sql.py
@@ -200,6 +200,27 @@ def query_result(datatable, ordered, query, spark):
         assert sorted(rows) == sorted(r)
 
 
+def _format_collected_value(value) -> str:
+    if value is None:
+        return "NULL"
+    if isinstance(value, bool):
+        return str(value).lower()
+    return str(value)
+
+
+@then(parsers.re("query result collected(?P( ordered)?)"))
+def query_result_collected(datatable, ordered, query, spark):
+    """Execute the SQL query with collect() and compare result with expected data table."""
+    expected_header, *expected_rows = datatable
+    df = spark.sql(query)
+    actual = [[_format_collected_value(value) for value in row] for row in df.collect()]
+    assert expected_header == df.schema.names
+    if ordered:
+        assert expected_rows == actual
+    else:
+        assert sorted(expected_rows) == sorted(actual)
+
+
 @then(parsers.parse("query error {error}"))
 def query_error(error, query, spark):
     """Executes the SQL query and expects it to fail with an error (regex match)."""
diff --git a/python/pysail/testing/spark/utils/jvm.py b/python/pysail/testing/spark/utils/jvm.py
new file mode 100644
index 000000000..0d1d60435
--- /dev/null
+++ b/python/pysail/testing/spark/utils/jvm.py
@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import contextlib
+import os
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+
+# Delta 4.1 embeds the Spark minor in its artifact name, while earlier releases
+# use the Scala binary version only.
+DELTA_SPARK_COORDINATES: dict[str, str] = {
+    "3.5": "io.delta:delta-spark_2.12:3.3.2",
+    "4.0": "io.delta:delta-spark_2.13:4.0.1",
+    "4.1": "io.delta:delta-spark_4.1_2.13:4.1.0",
+}
+
+
+def delta_spark_maven_coordinate(spark_version: str) -> str:
+    spark_minor = ".".join(spark_version.split(".")[:2])
+    try:
+        return DELTA_SPARK_COORDINATES[spark_minor]
+    except KeyError as error:
+        message = f"No Delta Maven coordinate mapping for Spark {spark_version}"
+        raise RuntimeError(message) from error
+
+
+@contextlib.contextmanager
+def classic_spark_mode() -> Generator[None, None, None]:
+    old_api_mode = os.environ.get("SPARK_API_MODE")
+    old_remote = os.environ.pop("SPARK_REMOTE", None)
+    old_connect_mode = os.environ.pop("SPARK_CONNECT_MODE_ENABLED", None)
+    os.environ["SPARK_API_MODE"] = "classic"
+    try:
+        yield
+    finally:
+        if old_api_mode is None:
+            os.environ.pop("SPARK_API_MODE", None)
+        else:
+            os.environ["SPARK_API_MODE"] = old_api_mode
+        if old_remote is not None:
+            os.environ["SPARK_REMOTE"] = old_remote
+        if old_connect_mode is not None:
+            os.environ["SPARK_CONNECT_MODE_ENABLED"] = old_connect_mode
diff --git a/python/pysail/tests/conftest.py b/python/pysail/tests/conftest.py
index cefecc8ac..430a2ead0 100644
--- a/python/pysail/tests/conftest.py
+++ b/python/pysail/tests/conftest.py
@@ -1,6 +1,7 @@
 import importlib
 import os
 import sys
+from pathlib import Path
 
 
 def pytest_configure(config):
@@ -38,6 +39,18 @@ def pytest_configure(config):
     configure_sail_environment()
 
 
+def pytest_collection_modifyitems(items):
+    # Add BDD feature file paths as an extra keyword to support test selection based on feature files.
+    package_root = Path(__file__).resolve().parents[1]
+    for item in items:
+        scenario = getattr(getattr(item, "function", None), "__scenario__", None)
+        feature = getattr(scenario, "feature", None)
+        filename = getattr(feature, "filename", None)
+        if filename:
+            path = Path(filename).resolve().relative_to(package_root)
+            item.extra_keyword_matches.add(path.as_posix())
+
+
 def configure_sail_environment():
     """Configure environment variables for PySail tests.
 
diff --git a/python/pysail/tests/spark/__snapshots__/test_clickbench.plan.yaml b/python/pysail/tests/spark/__snapshots__/test_clickbench.plan.yaml
index 80be7a7b3..6f9617d0d 100644
--- a/python/pysail/tests/spark/__snapshots__/test_clickbench.plan.yaml
+++ b/python/pysail/tests/spark/__snapshots__/test_clickbench.plan.yaml
@@ -312,7 +312,7 @@ data:
         RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
           SortPreservingMergeExec: [#109@4 ASC], fetch=10
             SortExec: TopK(fetch=10), expr=[#109@4 ASC], preserve_partitioning=[true]
-              ProjectionExec: expr=[WatchID@0 as #105, JavaEnable@1 as #106, Title@2 as #107, GoodEvent@3 as #108, CAST(EventTime@4 * 1000000 AS Timestamp(µs, "UTC")) as #109, 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) as #110, CounterID@6 as #111, ClientIP@7 as #112, RegionID@8 as #113, UserID@9 as #114, CounterClass@10 as #115, OS@11 as #116, UserAgent@12 as #117, URL@13 as #118, Referer@14 as #119, IsRefresh@15 as #120, RefererCategoryID@16 as #121, RefererRegionID@17 as #122, URLCategoryID@18 as #123, URLRegionID@19 as #124, ResolutionWidth@20 as #125, ResolutionHeight@21 as #126, ResolutionDepth@22 as #127, FlashMajor@23 as #128, FlashMinor@24 as #129, FlashMinor2@25 as #130, NetMajor@26 as #131, NetMinor@27 as #132, UserAgentMajor@28 as #133, UserAgentMinor@29 as #134, CookieEnable@30 as #135, JavascriptEnable@31 as #136, IsMobile@32 as #137, MobilePhone@33 as #138, MobilePhoneModel@34 as #139, Params@35 as #140, IPNetworkID@36 as #141, TraficSourceID@37 as #142, SearchEngineID@38 as #143, SearchPhrase@39 as #144, AdvEngineID@40 as #145, IsArtifical@41 as #146, WindowClientWidth@42 as #147, WindowClientHeight@43 as #148, ClientTimeZone@44 as #149, ClientEventTime@45 as #150, SilverlightVersion1@46 as #151, SilverlightVersion2@47 as #152, SilverlightVersion3@48 as #153, SilverlightVersion4@49 as #154, PageCharset@50 as #155, CodeVersion@51 as #156, IsLink@52 as #157, IsDownload@53 as #158, IsNotBounce@54 as #159, FUniqID@55 as #160, OriginalURL@56 as #161, HID@57 as #162, IsOldCounter@58 as #163, IsEvent@59 as #164, IsParameter@60 as #165, DontCountHits@61 as #166, WithHash@62 as #167, HitColor@63 as #168, LocalEventTime@64 as #169, Age@65 as #170, Sex@66 as #171, Income@67 as #172, Interests@68 as #173, Robotness@69 as #174, RemoteIP@70 as #175, WindowName@71 as #176, OpenerName@72 as #177, HistoryLength@73 as #178, BrowserLanguage@74 as #179, BrowserCountry@75 as #180, SocialNetwork@76 as #181, SocialAction@77 as #182, HTTPError@78 as #183, SendTiming@79 as #184, DNSTiming@80 as #185, ConnectTiming@81 as #186, ResponseStartTiming@82 as #187, ResponseEndTiming@83 as #188, FetchTiming@84 as #189, SocialSourceNetworkID@85 as #190, SocialSourcePage@86 as #191, ParamPrice@87 as #192, ParamOrderID@88 as #193, ParamCurrency@89 as #194, ParamCurrencyID@90 as #195, OpenstatServiceName@91 as #196, OpenstatCampaignID@92 as #197, OpenstatAdID@93 as #198, OpenstatSourceID@94 as #199, UTMSource@95 as #200, UTMMedium@96 as #201, UTMCampaign@97 as #202, UTMContent@98 as #203, UTMTerm@99 as #204, FromTag@100 as #205, HasGCLID@101 as #206, RefererHash@102 as #207, URLHash@103 as #208, CLID@104 as #209]
+              ProjectionExec: expr=[WatchID@0 as #105, JavaEnable@1 as #106, Title@2 as #107, GoodEvent@3 as #108, CAST(EventTime@4 * 1000000 AS Timestamp(µs, "UTC")) as #109, CAST(CAST(EventDate@5 AS Int32) AS Date32) as #110, CounterID@6 as #111, ClientIP@7 as #112, RegionID@8 as #113, UserID@9 as #114, CounterClass@10 as #115, OS@11 as #116, UserAgent@12 as #117, URL@13 as #118, Referer@14 as #119, IsRefresh@15 as #120, RefererCategoryID@16 as #121, RefererRegionID@17 as #122, URLCategoryID@18 as #123, URLRegionID@19 as #124, ResolutionWidth@20 as #125, ResolutionHeight@21 as #126, ResolutionDepth@22 as #127, FlashMajor@23 as #128, FlashMinor@24 as #129, FlashMinor2@25 as #130, NetMajor@26 as #131, NetMinor@27 as #132, UserAgentMajor@28 as #133, UserAgentMinor@29 as #134, CookieEnable@30 as #135, JavascriptEnable@31 as #136, IsMobile@32 as #137, MobilePhone@33 as #138, MobilePhoneModel@34 as #139, Params@35 as #140, IPNetworkID@36 as #141, TraficSourceID@37 as #142, SearchEngineID@38 as #143, SearchPhrase@39 as #144, AdvEngineID@40 as #145, IsArtifical@41 as #146, WindowClientWidth@42 as #147, WindowClientHeight@43 as #148, ClientTimeZone@44 as #149, ClientEventTime@45 as #150, SilverlightVersion1@46 as #151, SilverlightVersion2@47 as #152, SilverlightVersion3@48 as #153, SilverlightVersion4@49 as #154, PageCharset@50 as #155, CodeVersion@51 as #156, IsLink@52 as #157, IsDownload@53 as #158, IsNotBounce@54 as #159, FUniqID@55 as #160, OriginalURL@56 as #161, HID@57 as #162, IsOldCounter@58 as #163, IsEvent@59 as #164, IsParameter@60 as #165, DontCountHits@61 as #166, WithHash@62 as #167, HitColor@63 as #168, LocalEventTime@64 as #169, Age@65 as #170, Sex@66 as #171, Income@67 as #172, Interests@68 as #173, Robotness@69 as #174, RemoteIP@70 as #175, WindowName@71 as #176, OpenerName@72 as #177, HistoryLength@73 as #178, BrowserLanguage@74 as #179, BrowserCountry@75 as #180, SocialNetwork@76 as #181, SocialAction@77 as #182, HTTPError@78 as #183, SendTiming@79 as #184, DNSTiming@80 as #185, ConnectTiming@81 as #186, ResponseStartTiming@82 as #187, ResponseEndTiming@83 as #188, FetchTiming@84 as #189, SocialSourceNetworkID@85 as #190, SocialSourcePage@86 as #191, ParamPrice@87 as #192, ParamOrderID@88 as #193, ParamCurrency@89 as #194, ParamCurrencyID@90 as #195, OpenstatServiceName@91 as #196, OpenstatCampaignID@92 as #197, OpenstatAdID@93 as #198, OpenstatSourceID@94 as #199, UTMSource@95 as #200, UTMMedium@96 as #201, UTMCampaign@97 as #202, UTMContent@98 as #203, UTMTerm@99 as #204, FromTag@100 as #205, HasGCLID@101 as #206, RefererHash@102 as #207, URLHash@103 as #208, CLID@104 as #209]
                 FilterExec: CAST(URL@13 AS Utf8View) LIKE %google%
                   RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
                     DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ]
@@ -366,11 +366,11 @@ data:
     ProjectionExec: expr=[#105@0 as CounterID, #106@1 as l, #107@2 as c]
       SortPreservingMergeExec: [#106@1 DESC NULLS LAST], fetch=25
         SortExec: TopK(fetch=25), expr=[#106@1 DESC NULLS LAST], preserve_partitioning=[true]
-          ProjectionExec: expr=[#6@0 as #105, avg(character_length(hits.#13))@1 as #106, count(Int64(1))@2 as #107]
+          ProjectionExec: expr=[#6@0 as #105, avg(length(hits.#13))@1 as #106, count(Int64(1))@2 as #107]
             FilterExec: count(Int64(1))@2 > 100000
-              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[avg(character_length(hits.#13)), count(Int64(1))]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[avg(length(hits.#13)), count(Int64(1))]
                 RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=4
-                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[avg(character_length(hits.#13)), count(Int64(1))]
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[avg(length(hits.#13)), count(Int64(1))]
                     ProjectionExec: expr=[CounterID@0 as #6, URL@1 as #13]
                       FilterExec: URL@1 != 
                         RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
@@ -385,11 +385,11 @@ data:
         RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
           SortPreservingMergeExec: [#106@1 DESC NULLS LAST], fetch=25
             SortExec: TopK(fetch=25), expr=[#106@1 DESC NULLS LAST], preserve_partitioning=[true]
-              ProjectionExec: expr=[k@0 as #105, avg(character_length(hits.#14))@1 as #106, count(Int64(1))@2 as #107, min(hits.#14)@3 as #108]
+              ProjectionExec: expr=[k@0 as #105, avg(length(hits.#14))@1 as #106, count(Int64(1))@2 as #107, min(hits.#14)@3 as #108]
                 FilterExec: count(Int64(1))@2 > 100000
-                  AggregateExec: mode=FinalPartitioned, gby=[k@0 as k], aggr=[avg(character_length(hits.#14)), count(Int64(1)), min(hits.#14)]
+                  AggregateExec: mode=FinalPartitioned, gby=[k@0 as k], aggr=[avg(length(hits.#14)), count(Int64(1)), min(hits.#14)]
                     RepartitionExec: partitioning=Hash([k@0], 4), input_partitions=4
-                      AggregateExec: mode=Partial, gby=[regexp_replace(CAST(#14@0 AS LargeUtf8), ^https?:/(?:www.)?([^/]+)/.*$, $1, g) as k], aggr=[avg(character_length(hits.#14)), count(Int64(1)), min(hits.#14)]
+                      AggregateExec: mode=Partial, gby=[regexp_replace(CAST(#14@0 AS LargeUtf8), ^https?:/(?:www.)?([^/]+)/.*$, ${1}) as k], aggr=[avg(length(hits.#14)), count(Int64(1)), min(hits.#14)]
                         ProjectionExec: expr=[Referer@0 as #14]
                           FilterExec: Referer@0 != 
                             RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
@@ -400,9 +400,9 @@ data:
   |-
     == Physical Plan ==
     ProjectionExec: expr=[#105@0 as SUM(ResolutionWidth), #106@1 as SUM((ResolutionWidth + 1)), #107@2 as SUM((ResolutionWidth + 2)), #108@3 as SUM((ResolutionWidth + 3)), #109@4 as SUM((ResolutionWidth + 4)), #110@5 as SUM((ResolutionWidth + 5)), #111@6 as SUM((ResolutionWidth + 6)), #112@7 as SUM((ResolutionWidth + 7)), #113@8 as SUM((ResolutionWidth + 8)), #114@9 as SUM((ResolutionWidth + 9)), #115@10 as SUM((ResolutionWidth + 10)), #116@11 as SUM((ResolutionWidth + 11)), #117@12 as SUM((ResolutionWidth + 12)), #118@13 as SUM((ResolutionWidth + 13)), #119@14 as SUM((ResolutionWidth + 14)), #120@15 as SUM((ResolutionWidth + 15)), #121@16 as SUM((ResolutionWidth + 16)), #122@17 as SUM((ResolutionWidth + 17)), #123@18 as SUM((ResolutionWidth + 18)), #124@19 as SUM((ResolutionWidth + 19)), #125@20 as SUM((ResolutionWidth + 20)), #126@21 as SUM((ResolutionWidth + 21)), #127@22 as SUM((ResolutionWidth + 22)), #128@23 as SUM((ResolutionWidth + 23)), #129@24 as SUM((ResolutionWidth + 24)), #130@25 as SUM((ResolutionWidth + 25)), #131@26 as SUM((ResolutionWidth + 26)), #132@27 as SUM((ResolutionWidth + 27)), #133@28 as SUM((ResolutionWidth + 28)), #134@29 as SUM((ResolutionWidth + 29)), #135@30 as SUM((ResolutionWidth + 30)), #136@31 as SUM((ResolutionWidth + 31)), #137@32 as SUM((ResolutionWidth + 32)), #138@33 as SUM((ResolutionWidth + 33)), #139@34 as SUM((ResolutionWidth + 34)), #140@35 as SUM((ResolutionWidth + 35)), #141@36 as SUM((ResolutionWidth + 36)), #142@37 as SUM((ResolutionWidth + 37)), #143@38 as SUM((ResolutionWidth + 38)), #144@39 as SUM((ResolutionWidth + 39)), #145@40 as SUM((ResolutionWidth + 40)), #146@41 as SUM((ResolutionWidth + 41)), #147@42 as SUM((ResolutionWidth + 42)), #148@43 as SUM((ResolutionWidth + 43)), #149@44 as SUM((ResolutionWidth + 44)), #150@45 as SUM((ResolutionWidth + 45)), #151@46 as SUM((ResolutionWidth + 46)), #152@47 as SUM((ResolutionWidth + 47)), #153@48 as SUM((ResolutionWidth + 48)), #154@49 as SUM((ResolutionWidth + 49)), #155@50 as SUM((ResolutionWidth + 50)), #156@51 as SUM((ResolutionWidth + 51)), #157@52 as SUM((ResolutionWidth + 52)), #158@53 as SUM((ResolutionWidth + 53)), #159@54 as SUM((ResolutionWidth + 54)), #160@55 as SUM((ResolutionWidth + 55)), #161@56 as SUM((ResolutionWidth + 56)), #162@57 as SUM((ResolutionWidth + 57)), #163@58 as SUM((ResolutionWidth + 58)), #164@59 as SUM((ResolutionWidth + 59)), #165@60 as SUM((ResolutionWidth + 60)), #166@61 as SUM((ResolutionWidth + 61)), #167@62 as SUM((ResolutionWidth + 62)), #168@63 as SUM((ResolutionWidth + 63)), #169@64 as SUM((ResolutionWidth + 64)), #170@65 as SUM((ResolutionWidth + 65)), #171@66 as SUM((ResolutionWidth + 66)), #172@67 as SUM((ResolutionWidth + 67)), #173@68 as SUM((ResolutionWidth + 68)), #174@69 as SUM((ResolutionWidth + 69)), #175@70 as SUM((ResolutionWidth + 70)), #176@71 as SUM((ResolutionWidth + 71)), #177@72 as SUM((ResolutionWidth + 72)), #178@73 as SUM((ResolutionWidth + 73)), #179@74 as SUM((ResolutionWidth + 74)), #180@75 as SUM((ResolutionWidth + 75)), #181@76 as SUM((ResolutionWidth + 76)), #182@77 as SUM((ResolutionWidth + 77)), #183@78 as SUM((ResolutionWidth + 78)), #184@79 as SUM((ResolutionWidth + 79)), #185@80 as SUM((ResolutionWidth + 80)), #186@81 as SUM((ResolutionWidth + 81)), #187@82 as SUM((ResolutionWidth + 82)), #188@83 as SUM((ResolutionWidth + 83)), #189@84 as SUM((ResolutionWidth + 84)), #190@85 as SUM((ResolutionWidth + 85)), #191@86 as SUM((ResolutionWidth + 86)), #192@87 as SUM((ResolutionWidth + 87)), #193@88 as SUM((ResolutionWidth + 88)), #194@89 as SUM((ResolutionWidth + 89))]
-      ProjectionExec: expr=[sum(hits.#20)@0 as #105, sum(hits.#20 + Int32(1))@1 as #106, sum(hits.#20 + Int32(2))@2 as #107, sum(hits.#20 + Int32(3))@3 as #108, sum(hits.#20 + Int32(4))@4 as #109, sum(hits.#20 + Int32(5))@5 as #110, sum(hits.#20 + Int32(6))@6 as #111, sum(hits.#20 + Int32(7))@7 as #112, sum(hits.#20 + Int32(8))@8 as #113, sum(hits.#20 + Int32(9))@9 as #114, sum(hits.#20 + Int32(10))@10 as #115, sum(hits.#20 + Int32(11))@11 as #116, sum(hits.#20 + Int32(12))@12 as #117, sum(hits.#20 + Int32(13))@13 as #118, sum(hits.#20 + Int32(14))@14 as #119, sum(hits.#20 + Int32(15))@15 as #120, sum(hits.#20 + Int32(16))@16 as #121, sum(hits.#20 + Int32(17))@17 as #122, sum(hits.#20 + Int32(18))@18 as #123, sum(hits.#20 + Int32(19))@19 as #124, sum(hits.#20 + Int32(20))@20 as #125, sum(hits.#20 + Int32(21))@21 as #126, sum(hits.#20 + Int32(22))@22 as #127, sum(hits.#20 + Int32(23))@23 as #128, sum(hits.#20 + Int32(24))@24 as #129, sum(hits.#20 + Int32(25))@25 as #130, sum(hits.#20 + Int32(26))@26 as #131, sum(hits.#20 + Int32(27))@27 as #132, sum(hits.#20 + Int32(28))@28 as #133, sum(hits.#20 + Int32(29))@29 as #134, sum(hits.#20 + Int32(30))@30 as #135, sum(hits.#20 + Int32(31))@31 as #136, sum(hits.#20 + Int32(32))@32 as #137, sum(hits.#20 + Int32(33))@33 as #138, sum(hits.#20 + Int32(34))@34 as #139, sum(hits.#20 + Int32(35))@35 as #140, sum(hits.#20 + Int32(36))@36 as #141, sum(hits.#20 + Int32(37))@37 as #142, sum(hits.#20 + Int32(38))@38 as #143, sum(hits.#20 + Int32(39))@39 as #144, sum(hits.#20 + Int32(40))@40 as #145, sum(hits.#20 + Int32(41))@41 as #146, sum(hits.#20 + Int32(42))@42 as #147, sum(hits.#20 + Int32(43))@43 as #148, sum(hits.#20 + Int32(44))@44 as #149, sum(hits.#20 + Int32(45))@45 as #150, sum(hits.#20 + Int32(46))@46 as #151, sum(hits.#20 + Int32(47))@47 as #152, sum(hits.#20 + Int32(48))@48 as #153, sum(hits.#20 + Int32(49))@49 as #154, sum(hits.#20 + Int32(50))@50 as #155, sum(hits.#20 + Int32(51))@51 as #156, sum(hits.#20 + Int32(52))@52 as #157, sum(hits.#20 + Int32(53))@53 as #158, sum(hits.#20 + Int32(54))@54 as #159, sum(hits.#20 + Int32(55))@55 as #160, sum(hits.#20 + Int32(56))@56 as #161, sum(hits.#20 + Int32(57))@57 as #162, sum(hits.#20 + Int32(58))@58 as #163, sum(hits.#20 + Int32(59))@59 as #164, sum(hits.#20 + Int32(60))@60 as #165, sum(hits.#20 + Int32(61))@61 as #166, sum(hits.#20 + Int32(62))@62 as #167, sum(hits.#20 + Int32(63))@63 as #168, sum(hits.#20 + Int32(64))@64 as #169, sum(hits.#20 + Int32(65))@65 as #170, sum(hits.#20 + Int32(66))@66 as #171, sum(hits.#20 + Int32(67))@67 as #172, sum(hits.#20 + Int32(68))@68 as #173, sum(hits.#20 + Int32(69))@69 as #174, sum(hits.#20 + Int32(70))@70 as #175, sum(hits.#20 + Int32(71))@71 as #176, sum(hits.#20 + Int32(72))@72 as #177, sum(hits.#20 + Int32(73))@73 as #178, sum(hits.#20 + Int32(74))@74 as #179, sum(hits.#20 + Int32(75))@75 as #180, sum(hits.#20 + Int32(76))@76 as #181, sum(hits.#20 + Int32(77))@77 as #182, sum(hits.#20 + Int32(78))@78 as #183, sum(hits.#20 + Int32(79))@79 as #184, sum(hits.#20 + Int32(80))@80 as #185, sum(hits.#20 + Int32(81))@81 as #186, sum(hits.#20 + Int32(82))@82 as #187, sum(hits.#20 + Int32(83))@83 as #188, sum(hits.#20 + Int32(84))@84 as #189, sum(hits.#20 + Int32(85))@85 as #190, sum(hits.#20 + Int32(86))@86 as #191, sum(hits.#20 + Int32(87))@87 as #192, sum(hits.#20 + Int32(88))@88 as #193, sum(hits.#20 + Int32(89))@89 as #194]
-        AggregateExec: mode=Single, gby=[], aggr=[sum(hits.#20), sum(hits.#20 + Int32(1)), sum(hits.#20 + Int32(2)), sum(hits.#20 + Int32(3)), sum(hits.#20 + Int32(4)), sum(hits.#20 + Int32(5)), sum(hits.#20 + Int32(6)), sum(hits.#20 + Int32(7)), sum(hits.#20 + Int32(8)), sum(hits.#20 + Int32(9)), sum(hits.#20 + Int32(10)), sum(hits.#20 + Int32(11)), sum(hits.#20 + Int32(12)), sum(hits.#20 + Int32(13)), sum(hits.#20 + Int32(14)), sum(hits.#20 + Int32(15)), sum(hits.#20 + Int32(16)), sum(hits.#20 + Int32(17)), sum(hits.#20 + Int32(18)), sum(hits.#20 + Int32(19)), sum(hits.#20 + Int32(20)), sum(hits.#20 + Int32(21)), sum(hits.#20 + Int32(22)), sum(hits.#20 + Int32(23)), sum(hits.#20 + Int32(24)), sum(hits.#20 + Int32(25)), sum(hits.#20 + Int32(26)), sum(hits.#20 + Int32(27)), sum(hits.#20 + Int32(28)), sum(hits.#20 + Int32(29)), sum(hits.#20 + Int32(30)), sum(hits.#20 + Int32(31)), sum(hits.#20 + Int32(32)), sum(hits.#20 + Int32(33)), sum(hits.#20 + Int32(34)), sum(hits.#20 + Int32(35)), sum(hits.#20 + Int32(36)), sum(hits.#20 + Int32(37)), sum(hits.#20 + Int32(38)), sum(hits.#20 + Int32(39)), sum(hits.#20 + Int32(40)), sum(hits.#20 + Int32(41)), sum(hits.#20 + Int32(42)), sum(hits.#20 + Int32(43)), sum(hits.#20 + Int32(44)), sum(hits.#20 + Int32(45)), sum(hits.#20 + Int32(46)), sum(hits.#20 + Int32(47)), sum(hits.#20 + Int32(48)), sum(hits.#20 + Int32(49)), sum(hits.#20 + Int32(50)), sum(hits.#20 + Int32(51)), sum(hits.#20 + Int32(52)), sum(hits.#20 + Int32(53)), sum(hits.#20 + Int32(54)), sum(hits.#20 + Int32(55)), sum(hits.#20 + Int32(56)), sum(hits.#20 + Int32(57)), sum(hits.#20 + Int32(58)), sum(hits.#20 + Int32(59)), sum(hits.#20 + Int32(60)), sum(hits.#20 + Int32(61)), sum(hits.#20 + Int32(62)), sum(hits.#20 + Int32(63)), sum(hits.#20 + Int32(64)), sum(hits.#20 + Int32(65)), sum(hits.#20 + Int32(66)), sum(hits.#20 + Int32(67)), sum(hits.#20 + Int32(68)), sum(hits.#20 + Int32(69)), sum(hits.#20 + Int32(70)), sum(hits.#20 + Int32(71)), sum(hits.#20 + Int32(72)), sum(hits.#20 + Int32(73)), sum(hits.#20 + Int32(74)), sum(hits.#20 + Int32(75)), sum(hits.#20 + Int32(76)), sum(hits.#20 + Int32(77)), sum(hits.#20 + Int32(78)), sum(hits.#20 + Int32(79)), sum(hits.#20 + Int32(80)), sum(hits.#20 + Int32(81)), sum(hits.#20 + Int32(82)), sum(hits.#20 + Int32(83)), sum(hits.#20 + Int32(84)), sum(hits.#20 + Int32(85)), sum(hits.#20 + Int32(86)), sum(hits.#20 + Int32(87)), sum(hits.#20 + Int32(88)), sum(hits.#20 + Int32(89))]
-          DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[CAST(ResolutionWidth@20 AS Int32) as __common_expr_1, ResolutionWidth@20 as #20], file_type=parquet
+      ProjectionExec: expr=[sum(hits.#20)@0 as #105, sum(hits.#20)@0 + count(hits.#20)@1 as #106, sum(hits.#20)@0 + 2 * count(hits.#20)@1 as #107, sum(hits.#20)@0 + 3 * count(hits.#20)@1 as #108, sum(hits.#20)@0 + 4 * count(hits.#20)@1 as #109, sum(hits.#20)@0 + 5 * count(hits.#20)@1 as #110, sum(hits.#20)@0 + 6 * count(hits.#20)@1 as #111, sum(hits.#20)@0 + 7 * count(hits.#20)@1 as #112, sum(hits.#20)@0 + 8 * count(hits.#20)@1 as #113, sum(hits.#20)@0 + 9 * count(hits.#20)@1 as #114, sum(hits.#20)@0 + 10 * count(hits.#20)@1 as #115, sum(hits.#20)@0 + 11 * count(hits.#20)@1 as #116, sum(hits.#20)@0 + 12 * count(hits.#20)@1 as #117, sum(hits.#20)@0 + 13 * count(hits.#20)@1 as #118, sum(hits.#20)@0 + 14 * count(hits.#20)@1 as #119, sum(hits.#20)@0 + 15 * count(hits.#20)@1 as #120, sum(hits.#20)@0 + 16 * count(hits.#20)@1 as #121, sum(hits.#20)@0 + 17 * count(hits.#20)@1 as #122, sum(hits.#20)@0 + 18 * count(hits.#20)@1 as #123, sum(hits.#20)@0 + 19 * count(hits.#20)@1 as #124, sum(hits.#20)@0 + 20 * count(hits.#20)@1 as #125, sum(hits.#20)@0 + 21 * count(hits.#20)@1 as #126, sum(hits.#20)@0 + 22 * count(hits.#20)@1 as #127, sum(hits.#20)@0 + 23 * count(hits.#20)@1 as #128, sum(hits.#20)@0 + 24 * count(hits.#20)@1 as #129, sum(hits.#20)@0 + 25 * count(hits.#20)@1 as #130, sum(hits.#20)@0 + 26 * count(hits.#20)@1 as #131, sum(hits.#20)@0 + 27 * count(hits.#20)@1 as #132, sum(hits.#20)@0 + 28 * count(hits.#20)@1 as #133, sum(hits.#20)@0 + 29 * count(hits.#20)@1 as #134, sum(hits.#20)@0 + 30 * count(hits.#20)@1 as #135, sum(hits.#20)@0 + 31 * count(hits.#20)@1 as #136, sum(hits.#20)@0 + 32 * count(hits.#20)@1 as #137, sum(hits.#20)@0 + 33 * count(hits.#20)@1 as #138, sum(hits.#20)@0 + 34 * count(hits.#20)@1 as #139, sum(hits.#20)@0 + 35 * count(hits.#20)@1 as #140, sum(hits.#20)@0 + 36 * count(hits.#20)@1 as #141, sum(hits.#20)@0 + 37 * count(hits.#20)@1 as #142, sum(hits.#20)@0 + 38 * count(hits.#20)@1 as #143, sum(hits.#20)@0 + 39 * count(hits.#20)@1 as #144, sum(hits.#20)@0 + 40 * count(hits.#20)@1 as #145, sum(hits.#20)@0 + 41 * count(hits.#20)@1 as #146, sum(hits.#20)@0 + 42 * count(hits.#20)@1 as #147, sum(hits.#20)@0 + 43 * count(hits.#20)@1 as #148, sum(hits.#20)@0 + 44 * count(hits.#20)@1 as #149, sum(hits.#20)@0 + 45 * count(hits.#20)@1 as #150, sum(hits.#20)@0 + 46 * count(hits.#20)@1 as #151, sum(hits.#20)@0 + 47 * count(hits.#20)@1 as #152, sum(hits.#20)@0 + 48 * count(hits.#20)@1 as #153, sum(hits.#20)@0 + 49 * count(hits.#20)@1 as #154, sum(hits.#20)@0 + 50 * count(hits.#20)@1 as #155, sum(hits.#20)@0 + 51 * count(hits.#20)@1 as #156, sum(hits.#20)@0 + 52 * count(hits.#20)@1 as #157, sum(hits.#20)@0 + 53 * count(hits.#20)@1 as #158, sum(hits.#20)@0 + 54 * count(hits.#20)@1 as #159, sum(hits.#20)@0 + 55 * count(hits.#20)@1 as #160, sum(hits.#20)@0 + 56 * count(hits.#20)@1 as #161, sum(hits.#20)@0 + 57 * count(hits.#20)@1 as #162, sum(hits.#20)@0 + 58 * count(hits.#20)@1 as #163, sum(hits.#20)@0 + 59 * count(hits.#20)@1 as #164, sum(hits.#20)@0 + 60 * count(hits.#20)@1 as #165, sum(hits.#20)@0 + 61 * count(hits.#20)@1 as #166, sum(hits.#20)@0 + 62 * count(hits.#20)@1 as #167, sum(hits.#20)@0 + 63 * count(hits.#20)@1 as #168, sum(hits.#20)@0 + 64 * count(hits.#20)@1 as #169, sum(hits.#20)@0 + 65 * count(hits.#20)@1 as #170, sum(hits.#20)@0 + 66 * count(hits.#20)@1 as #171, sum(hits.#20)@0 + 67 * count(hits.#20)@1 as #172, sum(hits.#20)@0 + 68 * count(hits.#20)@1 as #173, sum(hits.#20)@0 + 69 * count(hits.#20)@1 as #174, sum(hits.#20)@0 + 70 * count(hits.#20)@1 as #175, sum(hits.#20)@0 + 71 * count(hits.#20)@1 as #176, sum(hits.#20)@0 + 72 * count(hits.#20)@1 as #177, sum(hits.#20)@0 + 73 * count(hits.#20)@1 as #178, sum(hits.#20)@0 + 74 * count(hits.#20)@1 as #179, sum(hits.#20)@0 + 75 * count(hits.#20)@1 as #180, sum(hits.#20)@0 + 76 * count(hits.#20)@1 as #181, sum(hits.#20)@0 + 77 * count(hits.#20)@1 as #182, sum(hits.#20)@0 + 78 * count(hits.#20)@1 as #183, sum(hits.#20)@0 + 79 * count(hits.#20)@1 as #184, sum(hits.#20)@0 + 80 * count(hits.#20)@1 as #185, sum(hits.#20)@0 + 81 * count(hits.#20)@1 as #186, sum(hits.#20)@0 + 82 * count(hits.#20)@1 as #187, sum(hits.#20)@0 + 83 * count(hits.#20)@1 as #188, sum(hits.#20)@0 + 84 * count(hits.#20)@1 as #189, sum(hits.#20)@0 + 85 * count(hits.#20)@1 as #190, sum(hits.#20)@0 + 86 * count(hits.#20)@1 as #191, sum(hits.#20)@0 + 87 * count(hits.#20)@1 as #192, sum(hits.#20)@0 + 88 * count(hits.#20)@1 as #193, sum(hits.#20)@0 + 89 * count(hits.#20)@1 as #194]
+        AggregateExec: mode=Single, gby=[], aggr=[sum(hits.#20), count(hits.#20)]
+          DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[CAST(ResolutionWidth@20 AS Int64) as __common_expr_2], file_type=parquet
 ---
 name: "test_clickbench_query_plan[30]"
 data:
@@ -492,9 +492,9 @@ data:
                   RepartitionExec: partitioning=Hash([#13@0], 4), input_partitions=4
                     AggregateExec: mode=Partial, gby=[#13@0 as #13], aggr=[count(Int64(1))]
                       ProjectionExec: expr=[URL@0 as #13]
-                        FilterExec: CounterID@1 = 62 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , projection=[URL@2]
+                        FilterExec: CounterID@1 = 62 AND EventDate@0 >= 15887 AND EventDate@0 <= 15917 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , projection=[URL@2]
                           RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                            DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND DontCountHits@61 = 0 AND IsRefresh@15 = 0 AND URL@13 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND URL_null_count@12 != row_count@3 AND (URL_min@10 !=  OR  != URL_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URL not in ()]
+                            DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15887 AND EventDate@5 <= 15917 AND DontCountHits@61 = 0 AND IsRefresh@15 = 0 AND URL@13 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15887 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15917 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND IsRefresh_null_count@12 != row_count@3 AND IsRefresh_min@10 <= 0 AND 0 <= IsRefresh_max@11 AND URL_null_count@15 != row_count@3 AND (URL_min@13 !=  OR  != URL_max@14), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URL not in ()]
 ---
 name: "test_clickbench_query_plan[37]"
 data:
@@ -510,9 +510,9 @@ data:
                   RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4
                     AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1))]
                       ProjectionExec: expr=[Title@0 as #2]
-                        FilterExec: CounterID@2 = 62 AND 1970-01-01 + CAST(CAST(EventDate@1 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@1 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , projection=[Title@0]
+                        FilterExec: CounterID@2 = 62 AND EventDate@1 >= 15887 AND EventDate@1 <= 15917 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , projection=[Title@0]
                           RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                            DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND DontCountHits@61 = 0 AND IsRefresh@15 = 0 AND Title@2 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND Title_null_count@12 != row_count@3 AND (Title_min@10 !=  OR  != Title_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), Title not in ()]
+                            DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15887 AND EventDate@5 <= 15917 AND DontCountHits@61 = 0 AND IsRefresh@15 = 0 AND Title@2 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15887 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15917 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND IsRefresh_null_count@12 != row_count@3 AND IsRefresh_min@10 <= 0 AND 0 <= IsRefresh_max@11 AND Title_null_count@15 != row_count@3 AND (Title_min@13 !=  OR  != Title_max@14), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), Title not in ()]
 ---
 name: "test_clickbench_query_plan[38]"
 data:
@@ -529,9 +529,9 @@ data:
                     RepartitionExec: partitioning=Hash([#13@0], 4), input_partitions=4
                       AggregateExec: mode=Partial, gby=[#13@0 as #13], aggr=[count(Int64(1))]
                         ProjectionExec: expr=[URL@0 as #13]
-                          FilterExec: CounterID@1 = 62 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, projection=[URL@2]
+                          FilterExec: CounterID@1 = 62 AND EventDate@0 >= 15887 AND EventDate@0 <= 15917 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, projection=[URL@2]
                             RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                              DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, URL, IsRefresh, IsLink, IsDownload], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@15 = 0 AND IsLink@52 != 0 AND IsDownload@53 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND IsLink_null_count@9 != row_count@3 AND (IsLink_min@7 != 0 OR 0 != IsLink_max@8) AND IsDownload_null_count@12 != row_count@3 AND IsDownload_min@10 <= 0 AND 0 <= IsDownload_max@11, required_guarantees=[CounterID in (62), IsDownload in (0), IsLink not in (0), IsRefresh in (0)]
+                              DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, URL, IsRefresh, IsLink, IsDownload], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15887 AND EventDate@5 <= 15917 AND IsRefresh@15 = 0 AND IsLink@52 != 0 AND IsDownload@53 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15887 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15917 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND IsLink_null_count@12 != row_count@3 AND (IsLink_min@10 != 0 OR 0 != IsLink_max@11) AND IsDownload_null_count@15 != row_count@3 AND IsDownload_min@13 <= 0 AND 0 <= IsDownload_max@14, required_guarantees=[CounterID in (62), IsDownload in (0), IsLink not in (0), IsRefresh in (0)]
 ---
 name: "test_clickbench_query_plan[39]"
 data:
@@ -548,9 +548,9 @@ data:
                     RepartitionExec: partitioning=Hash([#37@0, #38@1, #40@2, Src@3, Dst@4], 4), input_partitions=4
                       AggregateExec: mode=Partial, gby=[#37@2 as #37, #38@3 as #38, #40@4 as #40, CASE WHEN #38@3 = 0 AND #40@4 = 0 THEN #14@1 ELSE  END as Src, #13@0 as Dst], aggr=[count(Int64(1))]
                         ProjectionExec: expr=[URL@0 as #13, Referer@1 as #14, TraficSourceID@2 as #37, SearchEngineID@3 as #38, AdvEngineID@4 as #40]
-                          FilterExec: CounterID@1 = 62 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@4 = 0, projection=[URL@2, Referer@3, TraficSourceID@5, SearchEngineID@6, AdvEngineID@7]
+                          FilterExec: CounterID@1 = 62 AND EventDate@0 >= 15887 AND EventDate@0 <= 15917 AND IsRefresh@4 = 0, projection=[URL@2, Referer@3, TraficSourceID@5, SearchEngineID@6, AdvEngineID@7]
                             RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                              DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, URL, Referer, IsRefresh, TraficSourceID, SearchEngineID, AdvEngineID], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@15 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5, required_guarantees=[CounterID in (62), IsRefresh in (0)]
+                              DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, URL, Referer, IsRefresh, TraficSourceID, SearchEngineID, AdvEngineID], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15887 AND EventDate@5 <= 15917 AND IsRefresh@15 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15887 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15917 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8, required_guarantees=[CounterID in (62), IsRefresh in (0)]
 ---
 name: "test_clickbench_query_plan[40]"
 data:
@@ -564,10 +564,10 @@ data:
               AggregateExec: mode=FinalPartitioned, gby=[#103@0 as #103, #5@1 as #5], aggr=[count(Int64(1))]
                 RepartitionExec: partitioning=Hash([#103@0, #5@1], 4), input_partitions=4
                   AggregateExec: mode=Partial, gby=[#103@1 as #103, #5@0 as #5], aggr=[count(Int64(1))]
-                    ProjectionExec: expr=[1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) as #5, URLHash@1 as #103]
-                      FilterExec: CounterID@1 = 62 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5]
+                    ProjectionExec: expr=[CAST(CAST(EventDate@0 AS Int32) AS Date32) as #5, URLHash@1 as #103]
+                      FilterExec: CounterID@1 = 62 AND EventDate@0 >= 15887 AND EventDate@0 <= 15917 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5]
                         RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                          DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@15 = 0 AND (TraficSourceID@37 = -1 OR TraficSourceID@37 = 6) AND RefererHash@102 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND (TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= -1 AND -1 <= TraficSourceID_max@8 OR TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= 6 AND 6 <= TraficSourceID_max@8) AND RefererHash_null_count@12 != row_count@3 AND RefererHash_min@10 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@11, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)]
+                          DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15887 AND EventDate@5 <= 15917 AND IsRefresh@15 = 0 AND (TraficSourceID@37 = -1 OR TraficSourceID@37 = 6) AND RefererHash@102 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15887 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15917 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND (TraficSourceID_null_count@12 != row_count@3 AND TraficSourceID_min@10 <= -1 AND -1 <= TraficSourceID_max@11 OR TraficSourceID_null_count@12 != row_count@3 AND TraficSourceID_min@10 <= 6 AND 6 <= TraficSourceID_max@11) AND RefererHash_null_count@15 != row_count@3 AND RefererHash_min@13 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@14, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)]
 ---
 name: "test_clickbench_query_plan[41]"
 data:
@@ -582,9 +582,9 @@ data:
                 RepartitionExec: partitioning=Hash([#42@0, #43@1], 4), input_partitions=4
                   AggregateExec: mode=Partial, gby=[#42@0 as #42, #43@1 as #43], aggr=[count(Int64(1))]
                     ProjectionExec: expr=[WindowClientWidth@0 as #42, WindowClientHeight@1 as #43]
-                      FilterExec: CounterID@1 = 62 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@0 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, projection=[WindowClientWidth@3, WindowClientHeight@4]
+                      FilterExec: CounterID@1 = 62 AND EventDate@0 >= 15887 AND EventDate@0 <= 15917 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, projection=[WindowClientWidth@3, WindowClientHeight@4]
                         RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                          DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, IsRefresh, WindowClientWidth, WindowClientHeight, DontCountHits, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-01 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-31 AND IsRefresh@15 = 0 AND DontCountHits@61 = 0 AND URLHash@103 = 2868770270353813622, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND URLHash_null_count@12 != row_count@3 AND URLHash_min@10 <= 2868770270353813622 AND 2868770270353813622 <= URLHash_max@11, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URLHash in (2868770270353813622)]
+                          DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventDate, CounterID, IsRefresh, WindowClientWidth, WindowClientHeight, DontCountHits, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15887 AND EventDate@5 <= 15917 AND IsRefresh@15 = 0 AND DontCountHits@61 = 0 AND URLHash@103 = 2868770270353813622, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15887 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15917 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND DontCountHits_null_count@12 != row_count@3 AND DontCountHits_min@10 <= 0 AND 0 <= DontCountHits_max@11 AND URLHash_null_count@15 != row_count@3 AND URLHash_min@13 <= 2868770270353813622 AND 2868770270353813622 <= URLHash_max@14, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URLHash in (2868770270353813622)]
 ---
 name: "test_clickbench_query_plan[42]"
 data:
@@ -600,6 +600,6 @@ data:
                   RepartitionExec: partitioning=Hash([date_trunc(CASE WHEN Utf8("minute") ILIKE Utf8("mon") OR Utf8("minute") ILIKE Utf8("mm") THEN Utf8("month") WHEN Utf8("minute") ILIKE Utf8("yy") OR Utf8("minute") ILIKE Utf8("yyyy") THEN Utf8("year") WHEN Utf8("minute") ILIKE Utf8("dd") THEN Utf8("day") ELSE Utf8("minute") END,hits.#4)@0], 4), input_partitions=4
                     AggregateExec: mode=Partial, gby=[date_trunc(minute, #4@0) as date_trunc(CASE WHEN Utf8("minute") ILIKE Utf8("mon") OR Utf8("minute") ILIKE Utf8("mm") THEN Utf8("month") WHEN Utf8("minute") ILIKE Utf8("yy") OR Utf8("minute") ILIKE Utf8("yyyy") THEN Utf8("year") WHEN Utf8("minute") ILIKE Utf8("dd") THEN Utf8("day") ELSE Utf8("minute") END,hits.#4)], aggr=[count(Int64(1))]
                       ProjectionExec: expr=[CAST(EventTime@0 * 1000000 AS Timestamp(µs, "UTC")) as #4]
-                        FilterExec: CounterID@2 = 62 AND 1970-01-01 + CAST(CAST(EventDate@1 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-14 AND 1970-01-01 + CAST(CAST(EventDate@1 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-15 AND IsRefresh@3 = 0 AND DontCountHits@4 = 0, projection=[EventTime@0]
+                        FilterExec: CounterID@2 = 62 AND EventDate@1 >= 15900 AND EventDate@1 <= 15901 AND IsRefresh@3 = 0 AND DontCountHits@4 = 0, projection=[EventTime@0]
                           RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                            DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventTime, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) >= 2013-07-14 AND 1970-01-01 + CAST(CAST(EventDate@5 AS Utf8) ||  days AS Interval(DayTime)) <= 2013-07-15 AND IsRefresh@15 = 0 AND DontCountHits@61 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0)]
+                            DataSourceExec: file_groups={1 group: [[/hits.parquet]]}, projection=[EventTime, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND EventDate@5 >= 15900 AND EventDate@5 <= 15901 AND IsRefresh@15 = 0 AND DontCountHits@61 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND EventDate_max@4 >= 15900 AND EventDate_null_count@5 != row_count@3 AND EventDate_min@6 <= 15901 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND DontCountHits_null_count@12 != row_count@3 AND DontCountHits_min@10 <= 0 AND 0 <= DontCountHits_max@11, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0)]
diff --git a/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml b/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml
index e340bb157..57e4cc0cc 100644
--- a/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml
+++ b/python/pysail/tests/spark/__snapshots__/test_tpcds.plan.yaml
@@ -342,36 +342,35 @@ data:
                   AggregateExec: mode=FinalPartitioned, gby=[#8@0 as #8], aggr=[count(Int64(1))]
                     RepartitionExec: partitioning=Hash([#8@0], 4), input_partitions=4
                       AggregateExec: mode=Partial, gby=[#8@0 as #8], aggr=[count(Int64(1))]
-                        FilterExec: CAST(#87@1 AS Decimal128(11, 7)) > Some(12),2,1 * #155@2, projection=[#8@0]
-                          HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#94@2, #145@1)], projection=[#8@0, #87@1, #155@3]
-                            CoalescePartitionsExec
-                              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#82@0, #33@1)], projection=[#8@3, #87@1, #94@2]
-                                ProjectionExec: expr=[_0@0 as #82, _5@1 as #87, _12@2 as #94]
-                                  DataSourceExec: partitions=1, partition_sizes=[1]
-                                HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#31@1, CAST(d.#54 AS Float64)@1)], projection=[#8@0, #33@2]
-                                  CoalescePartitionsExec
-                                    HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.#13 AS Float64)@2, #34@2)], projection=[#8@0, #31@3, #33@4]
-                                      CoalescePartitionsExec
-                                        ProjectionExec: expr=[#8@0 as #8, #13@1 as #13, CAST(#13@1 AS Float64) as CAST(c.#13 AS Float64)]
-                                          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                                            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#17@1, #0@0)], projection=[#8@3, #13@0]
-                                              ProjectionExec: expr=[_0@0 as #13, _4@1 as #17]
-                                                DataSourceExec: partitions=1, partition_sizes=[1]
-                                              ProjectionExec: expr=[_0@0 as #0, _8@1 as #8]
-                                                DataSourceExec: partitions=1, partition_sizes=[1]
-                                      RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                                        ProjectionExec: expr=[_0@0 as #31, _2@1 as #33, _3@2 as #34]
-                                          DataSourceExec: partitions=1, partition_sizes=[1]
-                                  ProjectionExec: expr=[_0@0 as #54, CAST(_0@0 AS Float64) as CAST(d.#54 AS Float64)]
-                                    FilterExec: _3@1 = scalar_subquery(), projection=[_0@0]
-                                      RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                        HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#94@2, #145@1)], filter=CAST(#87@0 AS Decimal128(11, 7)) > Some(12),2,1 * #155@1, projection=[#8@0]
+                          CoalescePartitionsExec
+                            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#82@0, #33@1)], projection=[#8@3, #87@1, #94@2]
+                              ProjectionExec: expr=[_0@0 as #82, _5@1 as #87, _12@2 as #94]
+                                DataSourceExec: partitions=1, partition_sizes=[1]
+                              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#31@1, CAST(d.#54 AS Float64)@1)], projection=[#8@0, #33@2]
+                                CoalescePartitionsExec
+                                  HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.#13 AS Float64)@2, #34@2)], projection=[#8@0, #31@3, #33@4]
+                                    CoalescePartitionsExec
+                                      ProjectionExec: expr=[#8@0 as #8, #13@1 as #13, CAST(#13@1 AS Float64) as CAST(c.#13 AS Float64)]
+                                        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                                          HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#17@1, #0@0)], projection=[#8@3, #13@0]
+                                            ProjectionExec: expr=[_0@0 as #13, _4@1 as #17]
+                                              DataSourceExec: partitions=1, partition_sizes=[1]
+                                            ProjectionExec: expr=[_0@0 as #0, _8@1 as #8]
+                                              DataSourceExec: partitions=1, partition_sizes=[1]
+                                    RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                                      ProjectionExec: expr=[_0@0 as #31, _2@1 as #33, _3@2 as #34]
                                         DataSourceExec: partitions=1, partition_sizes=[1]
-                            ProjectionExec: expr=[avg(j.#138)@1 as #155, #145@0 as #145]
-                              AggregateExec: mode=FinalPartitioned, gby=[#145@0 as #145], aggr=[avg(j.#138)]
-                                RepartitionExec: partitioning=Hash([#145@0], 4), input_partitions=1
-                                  AggregateExec: mode=Partial, gby=[#145@1 as #145], aggr=[avg(j.#138)]
-                                    ProjectionExec: expr=[_5@0 as #138, _12@1 as #145]
+                                ProjectionExec: expr=[_0@0 as #54, CAST(_0@0 AS Float64) as CAST(d.#54 AS Float64)]
+                                  FilterExec: _3@1 = scalar_subquery(), projection=[_0@0]
+                                    RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
                                       DataSourceExec: partitions=1, partition_sizes=[1]
+                          ProjectionExec: expr=[avg(j.#138)@1 as #155, #145@0 as #145]
+                            AggregateExec: mode=FinalPartitioned, gby=[#145@0 as #145], aggr=[avg(j.#138)]
+                              RepartitionExec: partitioning=Hash([#145@0], 4), input_partitions=1
+                                AggregateExec: mode=Partial, gby=[#145@1 as #145], aggr=[avg(j.#138)]
+                                  ProjectionExec: expr=[_5@0 as #138, _12@1 as #145]
+                                    DataSourceExec: partitions=1, partition_sizes=[1]
         AggregateExec: mode=FinalPartitioned, gby=[#132@0 as #132], aggr=[]
           RepartitionExec: partitioning=Hash([#132@0], 4), input_partitions=4
             AggregateExec: mode=Partial, gby=[#132@0 as #132], aggr=[]
@@ -5597,7 +5596,7 @@ data:
       ProjectionExec: expr=[#68@0 as #68, #69@1 as #69]
         SortPreservingMergeExec: [#1@2 ASC], fetch=100
           SortExec: TopK(fetch=100), expr=[#68@0 ASC], preserve_partitioning=[true]
-            ProjectionExec: expr=[#1@0 as #68, spark_concat(spark_concat(CASE WHEN #9@2 IS NOT NULL THEN #9@2 ELSE  END, , ), CASE WHEN #8@1 IS NOT NULL THEN #8@1 ELSE  END) as #69, #1@0 as #1]
+            ProjectionExec: expr=[#1@0 as #68, spark_concat(CASE WHEN #9@2 IS NOT NULL THEN #9@2 ELSE  END, , , CASE WHEN #8@1 IS NOT NULL THEN #8@1 ELSE  END) as #69, #1@0 as #1]
               RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
                 HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_demographics.#31 AS Float64)@4, #52@0)], projection=[#1@0, #8@1, #9@2, #31@3, #52@5]
                   CoalescePartitionsExec
diff --git a/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml b/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml
index a4dec45ab..f611952f1 100644
--- a/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml
+++ b/python/pysail/tests/spark/analyzer/__snapshots__/features/explain.yaml
@@ -359,6 +359,29 @@ data:
             AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
               ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
                 DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+      ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#4@0 as k, #5@1 as SUM(v)]
+      ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          StageInputExec: input=0, partitioning=Hash([#2@0], 4)
 ---
 name: "test_explain_cost_shows_logical_plans_with_stats"
 data:
diff --git a/python/pysail/tests/spark/catalog/conftest.py b/python/pysail/tests/spark/catalog/conftest.py
index acd5ff1b2..1f2774959 100644
--- a/python/pysail/tests/spark/catalog/conftest.py
+++ b/python/pysail/tests/spark/catalog/conftest.py
@@ -3,7 +3,7 @@
 from pysail.testing.spark.session import spark_connect_server
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="package")
 def remote():
     # The catalog tests rely on the default memory catalog of a particular name.
     # We explicitly configure it so that the tests are skipped when `SPARK_REMOTE` is set,
diff --git a/python/pysail/tests/spark/catalog/features/function.feature b/python/pysail/tests/spark/catalog/features/function.feature
new file mode 100644
index 000000000..112a45933
--- /dev/null
+++ b/python/pysail/tests/spark/catalog/features/function.feature
@@ -0,0 +1,90 @@
+Feature: Catalog function queries
+
+  Scenario: SHOW SYSTEM FUNCTIONS filters built-in functions
+    When query
+      """
+      SHOW SYSTEM FUNCTIONS LIKE 'to_date'
+      """
+    Then query result
+      | function |
+      | to_date  |
+
+  Scenario: SHOW USER FUNCTIONS excludes built-in functions
+    When query
+      """
+      SHOW USER FUNCTIONS LIKE 'to_date'
+      """
+    Then query result
+      | function |
+
+  Scenario: SHOW FUNCTIONS supports namespace and legacy pattern forms
+    When query
+      """
+      SHOW FUNCTIONS IN default LIKE 'to_date'
+      """
+    Then query result
+      | function |
+      | to_date  |
+
+    When query
+      """
+      SHOW FUNCTIONS LIKE default.to_date
+      """
+    Then query result
+      | function |
+      | to_date  |
+
+    When query
+      """
+      SHOW FUNCTIONS no_such_db.to_date
+      """
+    Then query result
+      | function |
+      | to_date  |
+
+  Scenario: SHOW FUNCTIONS rejects namespace with unquoted legacy pattern
+    When query
+      """
+      SHOW FUNCTIONS IN default LIKE to_date
+      """
+    Then query error (?i)(expected|parse|syntax|extra input)
+
+  Scenario: DESCRIBE FUNCTION returns usage for a built-in function
+    When query
+      """
+      DESCRIBE FUNCTION abs
+      """
+    Then query result ordered
+      | function_desc                                                             |
+      | Function: abs                                                             |
+      | Class: org.apache.spark.sql.catalyst.expressions.Abs                       |
+      | Usage: abs(expr) - Returns the absolute value of the numeric or interval value. |
+
+  Scenario: DESCRIBE FUNCTION supports string literal names
+    When query
+      """
+      DESC FUNCTION 'concat'
+      """
+    Then query result ordered
+      | function_desc                                                                    |
+      | Function: concat                                                                 |
+      | Class: org.apache.spark.sql.catalyst.expressions.Concat                           |
+      | Usage: concat(col1, col2, ..., colN) - Returns the concatenation of col1, col2, ..., colN. |
+
+  Scenario: DESCRIBE FUNCTION supports operator names
+    When query
+      """
+      DESCRIBE FUNCTION +
+      """
+    Then query result ordered
+      | function_desc                              |
+      | Function: +                                |
+      | Class: org.apache.spark.sql.catalyst.expressions.Add |
+      | Usage: expr1 + expr2 - Returns `expr1`+`expr2`. |
+
+  Scenario: DESCRIBE FUNCTION does not resolve qualified built-in names
+    When query
+      """
+      DESCRIBE FUNCTION default.to_date
+      """
+    Then query error (?i)(not found|function)
diff --git a/python/pysail/tests/spark/catalog/features/memory_database_drop.feature b/python/pysail/tests/spark/catalog/features/memory_database_drop.feature
new file mode 100644
index 000000000..e1b1af1f6
--- /dev/null
+++ b/python/pysail/tests/spark/catalog/features/memory_database_drop.feature
@@ -0,0 +1,96 @@
+Feature: Memory catalog database drop semantics
+
+  Scenario: Drop non-empty memory database without CASCADE rejects and keeps objects
+    Given final statement
+      """
+      DROP DATABASE IF EXISTS no_cascade_memory_db CASCADE
+      """
+    Given statement
+      """
+      CREATE DATABASE no_cascade_memory_db
+      """
+    Given statement
+      """
+      CREATE TABLE no_cascade_memory_db.tbl (id INT)
+      """
+    Given statement
+      """
+      INSERT INTO no_cascade_memory_db.tbl VALUES (1), (2)
+      """
+    Given statement
+      """
+      CREATE VIEW no_cascade_memory_db.vw AS SELECT * FROM no_cascade_memory_db.tbl
+      """
+    Given statement with error .*
+      """
+      DROP DATABASE no_cascade_memory_db
+      """
+    When query
+      """
+      SHOW TABLES IN no_cascade_memory_db LIKE '*'
+      """
+    Then query result
+      | database                 | tableName | isTemporary |
+      | no_cascade_memory_db     | tbl       | false       |
+      | no_cascade_memory_db     | vw        | false       |
+    When query
+      """
+      SELECT * FROM no_cascade_memory_db.vw ORDER BY id
+      """
+    Then query result ordered
+      | id |
+      | 1  |
+      | 2  |
+
+    When query
+      """
+      SELECT * FROM no_cascade_memory_db.tbl ORDER BY id
+      """
+    Then query result ordered
+      | id |
+      | 1  |
+      | 2  |
+
+  Scenario: Drop non-empty memory database with CASCADE removes it and contained objects
+    Given final statement
+      """
+      DROP DATABASE IF EXISTS cascade_memory_db CASCADE
+      """
+    Given statement
+      """
+      CREATE DATABASE cascade_memory_db
+      """
+    Given statement
+      """
+      CREATE TABLE cascade_memory_db.tbl (id INT)
+      """
+    Given statement
+      """
+      INSERT INTO cascade_memory_db.tbl VALUES (1), (2)
+      """
+    Given statement
+      """
+      CREATE VIEW cascade_memory_db.vw AS SELECT * FROM cascade_memory_db.tbl
+      """
+    Given statement
+      """
+      DROP DATABASE cascade_memory_db CASCADE
+      """
+    When query
+      """
+      SHOW DATABASES LIKE 'cascade_memory_db'
+      """
+    Then query result
+      | name | catalog | description | locationUri |
+
+    When query
+      """
+      SELECT * FROM cascade_memory_db.tbl
+      """
+    Then query error .*
+
+    When query
+      """
+      SELECT * FROM cascade_memory_db.vw
+      """
+    Then query error .*
diff --git a/python/pysail/tests/spark/catalog/glue/conftest.py b/python/pysail/tests/spark/catalog/glue/conftest.py
index cd25d3ac2..61a8f3eda 100644
--- a/python/pysail/tests/spark/catalog/glue/conftest.py
+++ b/python/pysail/tests/spark/catalog/glue/conftest.py
@@ -39,7 +39,9 @@ def moto_endpoint(moto_container: DockerContainer) -> str:
 @pytest.fixture(scope="module")
 def remote(moto_endpoint: str) -> Generator[str, None, None]:
     """Start Sail server with Glue catalog."""
-    catalog_config = f'[{{name="sail", type="glue", region="us-east-1", endpoint_url="{moto_endpoint}"}}]'
+    catalog_config = (
+        f'[{{name="sail", type="glue", catalog_id="123456789012", region="us-east-1", endpoint_url="{moto_endpoint}"}}]'
+    )
     with spark_connect_server(
         envs={
             "SAIL_CATALOG__LIST": catalog_config,
diff --git a/python/pysail/tests/spark/catalog/hms/conftest.py b/python/pysail/tests/spark/catalog/hms/conftest.py
index 50831a266..e0d253191 100644
--- a/python/pysail/tests/spark/catalog/hms/conftest.py
+++ b/python/pysail/tests/spark/catalog/hms/conftest.py
@@ -3,7 +3,6 @@
 
 from __future__ import annotations
 
-import contextlib
 import hashlib
 import os
 import re
@@ -20,6 +19,7 @@
 from testcontainers.core.waiting_utils import wait_for_logs
 
 from pysail.testing.spark.session import spark_connect_server, spark_session_factory
+from pysail.testing.spark.utils.jvm import classic_spark_mode, delta_spark_maven_coordinate
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -111,25 +111,6 @@ def _spark_s3_options(endpoint: str) -> dict[str, str]:
     }
 
 
-@contextlib.contextmanager
-def _classic_spark_mode() -> Generator[None, None, None]:
-    old_api_mode = os.environ.get("SPARK_API_MODE")
-    old_remote = os.environ.pop("SPARK_REMOTE", None)
-    old_connect_mode = os.environ.pop("SPARK_CONNECT_MODE_ENABLED", None)
-    os.environ["SPARK_API_MODE"] = "classic"
-    try:
-        yield
-    finally:
-        if old_api_mode is None:
-            os.environ.pop("SPARK_API_MODE", None)
-        else:
-            os.environ["SPARK_API_MODE"] = old_api_mode
-        if old_remote is not None:
-            os.environ["SPARK_REMOTE"] = old_remote
-        if old_connect_mode is not None:
-            os.environ["SPARK_CONNECT_MODE_ENABLED"] = old_connect_mode
-
-
 def _wait_for_port(host: str, port: int, timeout: float) -> None:
     """Block until ``host:port`` accepts a TCP connection."""
     deadline = time.monotonic() + timeout
@@ -362,26 +343,101 @@ def _wait_for_hms_catalog(remote: str) -> None:
 # ---------------------------------------------------------------------------
 
 
-@pytest.fixture(scope="module")
+# Derive the Spark/Scala-specific runtime jar coordinates from the installed
+# pyspark version so this works across the Spark matrix (3.5.x uses Scala 2.12,
+# 4.x uses Scala 2.13) instead of hardcoding a single Spark minor.
+import pyspark as _pyspark  # noqa: E402
+
+_SPARK_VERSION = _pyspark.__version__
+_SPARK_MAJOR, _SPARK_MINOR_PART = _SPARK_VERSION.split(".")[:2]
+_SCALA_BINARY = "2.12" if _SPARK_MAJOR == "3" else "2.13"
+_SPARK_MINOR = f"{_SPARK_MAJOR}.{_SPARK_MINOR_PART}"
+try:
+    _DELTA_SPARK_PACKAGE = delta_spark_maven_coordinate(_SPARK_VERSION)
+except RuntimeError as error:
+    pytest.skip(reason=str(error))
+# Avro is a built-in but *external* datasource in Spark 2.4+: the runtime jar must
+# be on the classpath for `USING AVRO`.
+_AVRO_SPARK_PACKAGE = f"org.apache.spark:spark-avro_{_SCALA_BINARY}:{_SPARK_VERSION}"
+# Iceberg runtime (latest stable 1.11.0); the artifact encodes the Spark minor and
+# Scala binary versions. Resolved cleanly from Maven Central for Spark 4.1.
+_ICEBERG_SPARK_PACKAGE = f"org.apache.iceberg:iceberg-spark-runtime-{_SPARK_MINOR}_{_SCALA_BINARY}:1.11.0"
+
+
+@pytest.fixture(scope="session")
 def jvm_spark(
-    spark: SparkSession,
     hms_s3_metastore_endpoint: str,
     hms_warehouse_dir: Path,
     hms_s3_endpoint: str,
 ) -> Generator[SparkSession, None, None]:
-    """Start classic reference Spark with Hive support and MinIO S3A wiring."""
-    del spark
+    """Single session-scoped JVM Spark with Hive + Delta + Iceberg loaded.
+
+    Consolidates the former ``jvm_spark`` and ``delta_jvm_spark`` module-scoped
+    fixtures into one session. ``SparkSession.getOrCreate()`` is a JVM singleton:
+    ``spark.sql.extensions`` is applied exactly once, at first context creation, and
+    a second fixture asking for a differently-extended session in the same process
+    silently receives the FIRST session with its own extensions dropped -- while some
+    runtime catalog config still leaks through, producing an inconsistent,
+    operation-dependent failure (e.g. ``DeltaCatalog`` set but no Delta SQL parser
+    extension, so ``MERGE``/``UPDATE``/generated columns break).
+
+    One session-scoped unified session makes the extension/catalog set deterministic
+    regardless of fixture or test ordering, which is the precondition for safely
+    layering format interop on top without spawning further fragile per-format
+    sessions.
+
+    Iceberg is always loaded (not opt-in): the runtime jar resolves cleanly from
+    Maven Central, the extension does not perturb non-Iceberg table resolution, and
+    the full HMS suite passes with it enabled. Keeping it always-on ensures the
+    Iceberg HMS interop path is exercised in CI rather than skipped by default.
+
+    Trade-off accepted: plain parquet round-trip tests now run under ``DeltaCatalog``
+    instead of the default Hive session catalog. ``DeltaCatalog`` delegates non-Delta
+    tables to the underlying Hive catalog, so parquet/csv/json/avro behaviour is
+    unchanged -- this is the same configuration the former ``delta_jvm_spark`` already
+    relied on.
+    """
     warehouse_uri = hms_warehouse_dir.as_uri()
 
-    with _classic_spark_mode():
+    extensions = [
+        "io.delta.sql.DeltaSparkSessionExtension",
+        "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+    ]
+    spark_packages = [
+        _DELTA_SPARK_PACKAGE,
+        "org.apache.hadoop:hadoop-aws:3.4.2",
+        # Avro is a built-in but *external* datasource in Spark 2.4+: the runtime jar
+        # must be on the classpath for `USING AVRO`.
+        _AVRO_SPARK_PACKAGE,
+        _ICEBERG_SPARK_PACKAGE,
+    ]
+
+    with classic_spark_mode():
         builder = (
             SparkSession.builder.master("local[1]")
-            .appName("hms-jvm-s3")
+            .appName("hms-jvm-unified")
             .config("spark.sql.catalogImplementation", "hive")
+            # Both extension sets coexist in one session; extensions load once at
+            # context creation, so this list is the single source of truth.
+            .config("spark.sql.extensions", ",".join(extensions))
+            # Delta keeps the default ``spark_catalog`` (proven by the former
+            # ``delta_jvm_spark``); ``DeltaCatalog`` delegates non-Delta tables to Hive.
+            .config(
+                "spark.sql.catalog.spark_catalog",
+                "org.apache.spark.sql.delta.catalog.DeltaCatalog",
+            )
+            # Iceberg gets a NAMED catalog so it never fights Delta over ``spark_catalog``
+            # ownership. It points at the same HMS thrift endpoint; both engines share
+            # table registrations and the S3 (MinIO) metadata files.
+            .config("spark.sql.catalog.iceberg", "org.apache.iceberg.spark.SparkCatalog")
+            .config("spark.sql.catalog.iceberg.type", "hive")
+            .config("spark.sql.catalog.iceberg.uri", f"thrift://{hms_s3_metastore_endpoint}")
+            .config("spark.sql.catalog.iceberg.warehouse", warehouse_uri)
             .config(
                 "spark.hadoop.hive.metastore.uris",
                 f"thrift://{hms_s3_metastore_endpoint}",
             )
+            .config("spark.jars.packages", ",".join(spark_packages))
             .config("spark.sql.warehouse.dir", warehouse_uri)
             .config(
                 "spark.hadoop.javax.jdo.option.ConnectionURL",
@@ -389,6 +445,8 @@ def jvm_spark(
             )
         )
         for key, value in _spark_s3_options(hms_s3_endpoint).items():
+            if key == "spark.jars.packages":
+                continue
             builder = builder.config(key, value)
         spark = builder.enableHiveSupport().getOrCreate()
         spark.conf.set("spark.sql.session.timeZone", "UTC")
diff --git a/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_dsv2_roundtrip.py b/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_dsv2_roundtrip.py
new file mode 100644
index 000000000..8e6a44c6d
--- /dev/null
+++ b/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_dsv2_roundtrip.py
@@ -0,0 +1,796 @@
+# ruff: noqa: PLR2004, S608
+"""Spark -> Sail roundtrip tests for Spark datasource v2 (DSV2) formats and Iceberg.
+
+The JVM reference Spark creates a table in HMS using a datasource format; Sail
+reads it back through its HMS catalog. This exercises the SerDe ``path`` handling
+in :func:`sail_catalog_hms::convert::table_location` for the full Spark
+datasource provider set that Sail supports in HMS: parquet, csv, json, avro, and
+delta. Iceberg and Hive-native textfile tables have separate contract tests
+because their HMS metadata contracts are intentionally different.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pysail.tests.spark.catalog.hms.conftest import (
+    _describe_column_comments,
+    _describe_extended_properties,
+    _reference_catalog_table,
+    _scala_option_to_string,
+)
+
+if TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+_DSV2_FORMATS = ["parquet", "csv", "json", "avro", "delta"]
+# Sail reads every DSV2 format above, but cannot *write* Avro: its writer raises
+# "Writer not implemented for this format". The reverse (Sail-write) parametrization
+# keeps Avro in the matrix as xfail with that real error, so a future Avro writer
+# landing turns the xfail green without touching the read-side coverage.
+_SAIL_WRITE_FORMATS = [
+    "parquet",
+    "csv",
+    "json",
+    pytest.param(
+        "avro",
+        marks=pytest.mark.xfail(
+            reason="Sail has no Avro writer ('Writer not implemented for this format')",
+            strict=True,
+        ),
+    ),
+    "delta",
+]
+# Formats whose authoritative table metadata (schema, column comments, user
+# TBLPROPERTIES) lives in HMS table parameters, so assertions reading that metadata
+# back through Sail are meaningful. Delta is deliberately excluded: the DeltaCatalog
+# keeps schema/column comments/user properties in the Delta transaction log, not in
+# HMS, and rejects ``ALTER TABLE ... SET LOCATION`` on a Delta table outright
+# (``DELTA_MISSING_DELTA_TABLE``). Those behaviors are Spark/Delta semantics, not
+# Sail gaps, so parametrizing Delta through the HMS-metadata assertions would assert
+# a contract Delta does not provide.
+_HMS_METADATA_FORMATS = [fmt for fmt in _DSV2_FORMATS if fmt != "delta"]
+
+
+def _assert_sail_describes_datasource_table(
+    spark: SparkSession,
+    table_fqn: str,
+    *,
+    provider: str,
+) -> None:
+    properties = _describe_extended_properties(spark, table_fqn)
+    assert properties.get("Type") == "EXTERNAL", properties
+    assert properties.get("Provider", "").lower() == provider, properties
+    assert properties.get("Location"), f"no location resolved for {table_fqn}"
+
+
+@pytest.mark.parametrize("fmt", _DSV2_FORMATS)
+def test_spark_creates_sail_reads_dsv2(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Reference Spark creates a DSV2 datasource table; Sail reads it back.
+
+    ``CREATE TABLE ... USING `` registers the table in HMS as a Spark datasource
+    table carrying ``spark.sql.sources.provider=`` and the location mirrored into
+    the SerDe ``path`` parameter. Sail must recognise the provider, resolve the
+    location via the SerDe path, and return the rows.
+    """
+    table_fqn = f"{hms_s3_database}.roundtrip_{fmt}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT,
+          name STRING
+        )
+        USING {fmt.upper()}
+        """
+    )
+    jvm_spark.sql(f"INSERT INTO {table_fqn} VALUES (1, 'alice'), (2, 'bob')")
+
+    ref_rows = jvm_spark.sql(f"SELECT * FROM {table_fqn} ORDER BY id").collect()
+    assert len(ref_rows) == 2, f"Reference Spark expected 2 rows, got {len(ref_rows)}"
+
+    _assert_sail_describes_datasource_table(spark, table_fqn, provider=fmt)
+    sail_rows = spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in sail_rows] == [(1, "alice"), (2, "bob")]
+
+
+def _assert_jvm_spark_sees_datasource_provider(
+    jvm_spark: SparkSession,
+    database: str,
+    table: str,
+    provider: str,
+) -> None:
+    """Assert that JVM Spark sees the expected provider in HMS metadata.
+
+    This is an independent cross-engine check: JVM Spark reads Sail's HMS metadata
+    directly via ``externalCatalog().getTable()``, not via DESCRIBE EXTENDED.
+    The comparison is case-insensitive because Spark normalizes the provider
+    string differently across versions (e.g. ``USING ORC`` is recorded as
+    ``ORC``), and the contract we care about is the *provider identity*, not its
+    casing.
+    """
+    spark_table = _reference_catalog_table(jvm_spark, database, table)
+    actual = _scala_option_to_string(spark_table.provider())
+    assert actual is not None, f"provider is None for {database}.{table}"
+    assert actual.lower() == provider.lower(), f"expected provider={provider!r}, got {actual!r}"
+
+
+def _reference_table_properties(jvm_spark: SparkSession, database: str, table: str) -> dict[str, str]:
+    scala_properties = _reference_catalog_table(jvm_spark, database, table).properties()
+    iterator = scala_properties.iterator()
+    properties: dict[str, str] = {}
+    while iterator.hasNext():
+        entry = iterator.next()
+        properties[str(entry.productElement(0))] = str(entry.productElement(1))
+    return properties
+
+
+def _set_hms_table_parameter(
+    reference_spark: SparkSession,
+    database: str,
+    table: str,
+    key: str,
+    value: str,
+) -> None:
+    """Update one table parameter directly so storage metadata stays unchanged."""
+    jvm = reference_spark._jvm  # noqa: SLF001
+    hadoop_conf = reference_spark._jsc.hadoopConfiguration()  # noqa: SLF001
+    hive_conf = jvm.org.apache.hadoop.hive.conf.HiveConf()
+    hive_conf.set("hive.metastore.uris", hadoop_conf.get("hive.metastore.uris"))
+    client = jvm.org.apache.hadoop.hive.metastore.HiveMetaStoreClient(hive_conf)
+    try:
+        hms_table = client.getTable(database, table)
+        hms_table.getParameters().put(key, value)
+        client.alter_table(database, table, hms_table)
+        actual = client.getTable(database, table).getParameters().get(key)
+        assert actual == value, f"expected raw HMS parameter {key}={value!r}, got {actual!r}"
+    finally:
+        client.close()
+
+
+@pytest.mark.parametrize("fmt", _SAIL_WRITE_FORMATS)
+def test_sail_creates_spark_reads_dsv2(
+    spark: SparkSession,
+    jvm_spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Sail creates a DSV2 datasource table; reference Spark reads it back.
+
+    The reverse direction: Sail's ``build_generic_table`` writes the SerDe ``path``
+    parameter (the change in this PR), and reference Spark must resolve the table
+    location and read the rows.
+    """
+    table_fqn = f"{hms_s3_database}.reverse_{fmt}"
+
+    spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT,
+          name STRING
+        )
+        USING {fmt.upper()}
+        """
+    )
+    spark.sql(f"INSERT INTO {table_fqn} VALUES (1, 'alice'), (2, 'bob')")
+
+    sail_rows = spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert len(sail_rows) == 2, f"Sail expected 2 rows, got {len(sail_rows)}"
+
+    # Verify Sail's own read of the metadata it wrote: this is the write-side
+    # assertion that directly exercises build_generic_table's provider + SerDe path
+    # handling (the change in this PR), not just row parity.
+    _assert_sail_describes_datasource_table(spark, table_fqn, provider=fmt)
+
+    # Independent cross-engine validation: JVM Spark reads HMS metadata directly.
+    _assert_jvm_spark_sees_datasource_provider(jvm_spark, hms_s3_database, f"reverse_{fmt}", provider=fmt)
+
+    ref_rows = jvm_spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in ref_rows] == [(1, "alice"), (2, "bob")]
+
+
+def test_spark_creates_sail_reads_iceberg(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Reference Spark creates an Iceberg table in HMS; Sail reads it back.
+
+    Spark's ``iceberg`` catalog (type=hive) registers the table in HMS with
+    ``table_type=ICEBERG`` and a ``metadata-location`` property. Sail detects the
+    Iceberg marker, resolves the format, loads the metadata file from S3 via
+    :class:`IcebergTableProvider`, and returns the rows.
+    """
+    table = "roundtrip_iceberg"
+    # Spark writes through its named ``iceberg`` catalog; Sail reads the same HMS
+    # table via the ``sail`` catalog (plain db.table, no catalog prefix).
+    spark_iceberg_fqn = f"iceberg.{hms_s3_database}.{table}"
+    sail_fqn = f"{hms_s3_database}.{table}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {spark_iceberg_fqn} (
+          id INT,
+          name STRING
+        )
+        USING ICEBERG
+        """
+    )
+    jvm_spark.sql(f"INSERT INTO {spark_iceberg_fqn} VALUES (1, 'alice'), (2, 'bob')")
+
+    ref_rows = jvm_spark.sql(f"SELECT id, name FROM {spark_iceberg_fqn} ORDER BY id").collect()
+    assert len(ref_rows) == 2, f"Reference Spark expected 2 rows, got {len(ref_rows)}"
+
+    properties = _describe_extended_properties(spark, sail_fqn)
+    assert properties.get("Provider", "").lower() == "iceberg", properties
+    assert properties.get("Location"), f"Sail did not resolve a location for {sail_fqn}"
+
+    sail_rows = spark.sql(f"SELECT id, name FROM {sail_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in sail_rows] == [(1, "alice"), (2, "bob")]
+
+
+def test_sail_creates_spark_reads_iceberg(
+    spark: SparkSession,
+    jvm_spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Sail creates an Iceberg table in HMS; reference Spark reads it back.
+
+    The reverse direction of ``test_spark_creates_sail_reads_iceberg``: Sail's
+    CatalogCoordinated create commits ``table_type=ICEBERG`` and a ``metadata-location``
+    to HMS. Reference Spark reads the table through its named ``iceberg`` catalog
+    (type=hive), which detects the Iceberg marker and loads the metadata file.
+    """
+    table = "reverse_iceberg"
+    sail_fqn = f"{hms_s3_database}.{table}"
+    spark_iceberg_fqn = f"iceberg.{hms_s3_database}.{table}"
+
+    spark.sql(
+        f"""
+        CREATE TABLE {sail_fqn} (
+          id INT,
+          name STRING
+        )
+        USING ICEBERG
+        """
+    )
+    spark.sql(f"INSERT INTO {sail_fqn} VALUES (1, 'alice'), (2, 'bob')")
+
+    # Sail reads its own metadata back: Provider=iceberg, location resolved.
+    properties = _describe_extended_properties(spark, sail_fqn)
+    assert properties.get("Provider", "").lower() == "iceberg", properties
+    assert properties.get("Location"), f"Sail did not resolve a location for {sail_fqn}"
+
+    sail_rows = spark.sql(f"SELECT id, name FROM {sail_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in sail_rows] == [(1, "alice"), (2, "bob")]
+
+    # Reference Spark reads the same HMS table through its iceberg catalog.
+    ref_rows = jvm_spark.sql(f"SELECT id, name FROM {spark_iceberg_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in ref_rows] == [(1, "alice"), (2, "bob")]
+
+
+def test_iceberg_append_visibility_both_ways(
+    spark: SparkSession,
+    jvm_spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Spark and Sail alternately append to one HMS Iceberg table and see snapshots."""
+    table = "append_iceberg"
+    spark_iceberg_fqn = f"iceberg.{hms_s3_database}.{table}"
+    sail_fqn = f"{hms_s3_database}.{table}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {spark_iceberg_fqn} (
+          id INT,
+          name STRING
+        )
+        USING ICEBERG
+        """
+    )
+    jvm_spark.sql(f"INSERT INTO {spark_iceberg_fqn} VALUES (1, 'spark')")
+    assert [(r.id, r.name) for r in spark.sql(f"SELECT id, name FROM {sail_fqn}").collect()] == [(1, "spark")]
+
+    spark.sql(f"INSERT INTO {sail_fqn} VALUES (2, 'sail')")
+    # JVM Spark's Iceberg ``SparkCatalog`` caches table metadata in-process for the
+    # session, so an external commit that advances the HMS ``metadata-location``
+    # pointer is not picked up automatically. An explicit ``REFRESH TABLE`` forces a
+    # re-read of the catalog entry. This is Spark catalog semantics, not a Sail gap:
+    # pyiceberg (``StaticTable.from_metadata``), which has no catalog cache, sees
+    # Sail's appended row without any refresh. Each engine that reads after another
+    # engine's external commit must refresh its own catalog view first.
+    jvm_spark.catalog.refreshTable(spark_iceberg_fqn)
+    ref_rows = jvm_spark.sql(f"SELECT id, name FROM {spark_iceberg_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in ref_rows] == [(1, "spark"), (2, "sail")]
+
+    jvm_spark.sql(f"INSERT INTO {spark_iceberg_fqn} VALUES (3, 'spark-again')")
+    # Sail re-reads HMS table metadata on each access (it does not hold an in-process
+    # catalog cache like the JVM Iceberg SparkCatalog), so no refresh is needed here.
+    sail_rows = spark.sql(f"SELECT id, name FROM {sail_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in sail_rows] == [(1, "spark"), (2, "sail"), (3, "spark-again")]
+
+
+def test_spark_creates_sail_reads_iceberg_type_properties_and_comments(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Iceberg HMS tables preserve basic schema fidelity, properties, and comments."""
+    table = "iceberg_types_props"
+    spark_iceberg_fqn = f"iceberg.{hms_s3_database}.{table}"
+    sail_fqn = f"{hms_s3_database}.{table}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {spark_iceberg_fqn} (
+          id INT COMMENT 'identifier',
+          amount DECIMAL(10, 2),
+          ts TIMESTAMP
+        )
+        USING ICEBERG
+        TBLPROPERTIES ('interop.owner' = 'spark')
+        """
+    )
+    jvm_spark.sql(
+        f"""
+        INSERT INTO {spark_iceberg_fqn} VALUES
+          (1, CAST(12.34 AS DECIMAL(10, 2)), TIMESTAMP '2024-01-02 03:04:05'),
+          (2, CAST(0.10 AS DECIMAL(10, 2)), TIMESTAMP '2024-01-02 03:04:06')
+        """
+    )
+
+    hms_properties = _reference_table_properties(jvm_spark, hms_s3_database, table)
+    assert hms_properties.get("table_type", "").lower() == "iceberg", hms_properties
+    assert hms_properties.get("metadata-location") or hms_properties.get("metadata_location"), hms_properties
+
+    sail_properties = _describe_extended_properties(spark, sail_fqn)
+    assert sail_properties.get("Provider", "").lower() == "iceberg", sail_properties
+    assert "interop.owner=spark" in sail_properties.get("Table Properties", ""), sail_properties
+    assert _describe_column_comments(spark, sail_fqn)["id"] == "identifier"
+
+    fields = {field.name: field for field in spark.table(sail_fqn).schema.fields}
+    assert fields["amount"].dataType.simpleString() == "decimal(10,2)", fields
+    assert fields["ts"].dataType.simpleString() == "timestamp", fields
+    rows = spark.sql(
+        f"""
+        SELECT id, CAST(amount AS STRING) AS amount, CAST(ts AS STRING) AS ts
+        FROM {sail_fqn}
+        ORDER BY id
+        """
+    ).collect()
+    assert [(r.id, r.amount, r.ts) for r in rows] == [
+        (1, "12.34", "2024-01-02 03:04:05"),
+        (2, "0.10", "2024-01-02 03:04:06"),
+    ]
+
+
+def test_spark_creates_sail_reads_partitioned_iceberg(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Spark writes an identity-partitioned HMS Iceberg table; Sail reads and prunes."""
+    table = "iceberg_part"
+    spark_iceberg_fqn = f"iceberg.{hms_s3_database}.{table}"
+    sail_fqn = f"{hms_s3_database}.{table}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {spark_iceberg_fqn} (
+          id INT,
+          name STRING,
+          event_date DATE
+        )
+        USING ICEBERG
+        PARTITIONED BY (event_date)
+        """
+    )
+    jvm_spark.sql(
+        f"""
+        INSERT INTO {spark_iceberg_fqn} VALUES
+          (1, 'alice', DATE '2024-01-02'),
+          (2, 'bob', DATE '2024-01-03'),
+          (3, 'cara', DATE '2024-01-02')
+        """
+    )
+
+    rows = spark.sql(
+        f"""
+        SELECT id, name, CAST(event_date AS STRING) AS event_date
+        FROM {sail_fqn}
+        WHERE event_date = DATE '2024-01-02'
+        ORDER BY id
+        """
+    ).collect()
+    assert [(r.id, r.name, r.event_date) for r in rows] == [
+        (1, "alice", "2024-01-02"),
+        (3, "cara", "2024-01-02"),
+    ]
+
+
+@pytest.mark.xfail(
+    reason=(
+        "Sail detects the Hive textfile format at the HMS metadata layer but has no "
+        "textfile file reader, so reading raises 'No table format found for: textfile'."
+    ),
+    strict=True,
+)
+def test_spark_creates_sail_reads_hive_textfile_table(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Hive SerDe textfile has no datasource provider and uses SD location."""
+    table = "native_textfile"
+    table_fqn = f"{hms_s3_database}.{table}"
+
+    jvm_spark.sql(f"CREATE TABLE {table_fqn} (value STRING) STORED AS TEXTFILE")
+    jvm_spark.sql(f"INSERT INTO {table_fqn} VALUES ('alpha'), ('beta')")
+
+    # A Hive-native textfile table carries no ``spark.sql.sources.provider`` key in
+    # HMS parameters; JVM ``CatalogTable.provider()`` still reports ``hive``, but that
+    # is the Hive catalog handle, not a Spark datasource provider. The real contract
+    # is the *absence* of the datasource provider key, which is what forces Sail to
+    # resolve the table via SerDe/storage-descriptor detection instead of the path.
+    hms_props = _reference_table_properties(jvm_spark, hms_s3_database, table)
+    assert "spark.sql.sources.provider" not in {k.lower() for k in hms_props}, hms_props
+    assert _scala_option_to_string(_reference_catalog_table(jvm_spark, hms_s3_database, table).storage().locationUri())
+
+    properties = _describe_extended_properties(spark, table_fqn)
+    assert properties.get("Provider", "").lower() == "textfile", properties
+    assert properties.get("Location"), properties
+    rows = spark.sql(f"SELECT value FROM {table_fqn} ORDER BY value").collect()
+    assert [row.value for row in rows] == ["alpha", "beta"]
+
+
+def test_unsupported_datasource_provider_does_not_poison_database_listing(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """An unsupported Spark datasource provider must not break database listing.
+
+    The JVM reference Spark creates ORC and Parquet datasource tables. The test
+    then changes a second Parquet table's provider marker to a custom provider
+    without changing its Parquet SerDe metadata. Sail must enumerate all three
+    and preserve both unsupported provider identities when describing them.
+
+    Listing/describe must stay usable for any provider: ``list_tables`` maps
+    every HMS table through ``table_to_status``, so an unsupported provider on
+    one table must not fail the whole listing. A datasource provider marker is
+    authoritative: Sail must not relabel ``com.acme.CustomSource`` as Parquet
+    merely because the provider uses Parquet storage metadata.
+
+    Reading the ORC table is intentionally not asserted: Sail may still fail
+    at the scan/read layer, which is acceptable as long as listing and describe
+    succeed.
+    """
+    orc_table = "orc_t"
+    parquet_table = "parquet_t"
+    custom_table = "custom_provider_t"
+    orc_fqn = f"{hms_s3_database}.{orc_table}"
+    parquet_fqn = f"{hms_s3_database}.{parquet_table}"
+    custom_fqn = f"{hms_s3_database}.{custom_table}"
+    custom_provider = "com.acme.CustomSource"
+
+    jvm_spark.sql(f"CREATE TABLE {orc_fqn} (id INT, name STRING) USING ORC")
+    jvm_spark.sql(f"CREATE TABLE {parquet_fqn} (id INT, name STRING) USING PARQUET")
+    jvm_spark.sql(f"CREATE TABLE {custom_fqn} (id INT, name STRING) USING PARQUET")
+    _set_hms_table_parameter(
+        jvm_spark,
+        hms_s3_database,
+        custom_table,
+        "spark.sql.sources.provider",
+        custom_provider,
+    )
+
+    # JVM cross-engine sanity: the ORC table really is a datasource table with
+    # provider=orc in HMS metadata (i.e. this test really exercises the
+    # unsupported-provider path, not a Hive-native SerDe-only table).
+    _assert_jvm_spark_sees_datasource_provider(jvm_spark, hms_s3_database, orc_table, provider="orc")
+
+    # Listing contract: both tables are visible through Sail's HMS catalog.
+    # An unsupported provider must not fail ``list_tables``.
+    shown = {row.tableName for row in spark.sql(f"SHOW TABLES IN {hms_s3_database}").collect()}
+    assert orc_table in shown, f"orc table missing from SHOW TABLES: {shown}"
+    assert parquet_table in shown, f"parquet table missing from SHOW TABLES: {shown}"
+    assert custom_table in shown, f"custom-provider table missing from SHOW TABLES: {shown}"
+
+    # The ORC table's metadata is visible and preserves its provider (Sail has
+    # no ORC reader, but metadata-only access must still work).
+    orc_properties = _describe_extended_properties(spark, orc_fqn)
+    assert orc_properties.get("Provider", "").lower() == "orc", orc_properties
+    assert orc_properties.get("Location"), orc_properties
+
+    custom_properties = _describe_extended_properties(spark, custom_fqn)
+    assert custom_properties.get("Provider", "").lower() == custom_provider.lower(), custom_properties
+    assert custom_properties.get("Location"), custom_properties
+
+
+# ---------------------------------------------------------------------------
+# Metadata and type fidelity for DSV2 formats (spark -> sail read path).
+#
+# These catch format-specific interop breaks that the basic (INT, STRING)
+# round-trip cannot surface: decimal scale/precision, timestamp serialisation,
+# SerDe-path updates on ALTER TABLE, foreign property visibility, and column
+# comments. All are spark->sail because that is the direction where Sail's reader
+# must interpret Spark's written files/metadata, and it is the direction that
+# works for every DSV2 format including avro.
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("fmt", _DSV2_FORMATS)
+def test_spark_creates_sail_reads_dsv2_type_fidelity(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Reference Spark writes DECIMAL and TIMESTAMP; Sail restores exact types and values."""
+    table_fqn = f"{hms_s3_database}.types_{fmt}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT,
+          amount DECIMAL(10, 2),
+          ts TIMESTAMP
+        )
+        USING {fmt.upper()}
+        """
+    )
+    jvm_spark.sql(
+        f"""
+        INSERT INTO {table_fqn} VALUES
+          (1, CAST(12.34 AS DECIMAL(10, 2)), TIMESTAMP '2024-01-02 03:04:05'),
+          (2, CAST(0.10 AS DECIMAL(10, 2)), TIMESTAMP '2024-01-02 03:04:06')
+        """
+    )
+
+    # Schema is preserved: decimal precision/scale and timestamp type survive.
+    schema = spark.table(table_fqn).schema
+    fields = {field.name: field for field in schema.fields}
+    assert fields["amount"].dataType.simpleString() == "decimal(10,2)", fields
+    assert fields["ts"].dataType.simpleString() == "timestamp", fields
+
+    # Values are restored exactly. Cast to string to avoid decimal/timezone
+    # comparison noise (mirrors the existing parquet timestamp test).
+    rows = spark.sql(
+        f"""
+        SELECT
+          id,
+          CAST(amount AS STRING) AS amount,
+          CAST(ts AS STRING) AS ts
+        FROM {table_fqn}
+        ORDER BY id
+        """
+    ).collect()
+    assert [(r.id, r.amount, r.ts) for r in rows] == [
+        (1, "12.34", "2024-01-02 03:04:05"),
+        (2, "0.10", "2024-01-02 03:04:06"),
+    ]
+
+
+@pytest.mark.parametrize("fmt", _HMS_METADATA_FORMATS)
+def test_spark_alters_dsv2_table_location_sail_reads_new_path(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Spark ALTER TABLE SET LOCATION updates the datasource SerDe path; Sail reads it.
+
+    Distinct from create: the SerDe ``path`` *update* path. After SET LOCATION the
+    old location is abandoned; Sail must resolve and read only the new location.
+    Mirrors the existing parquet ``test_spark_alters_datasource_table_location_*``.
+    """
+    table_fqn = f"{hms_s3_database}.alter_loc_{fmt}"
+    old_location = f"s3://hms-warehouse/{hms_s3_database}/alter_loc_{fmt}_old"
+    new_location = f"s3://hms-warehouse/{hms_s3_database}/alter_loc_{fmt}_new"
+
+    jvm_spark.sql(f"CREATE TABLE {table_fqn} (id INT, name STRING) USING {fmt.upper()} LOCATION '{old_location}'")
+    jvm_spark.sql(f"INSERT INTO {table_fqn} VALUES (1, 'old')")
+    jvm_spark.sql(f"ALTER TABLE {table_fqn} SET LOCATION '{new_location}'")
+    jvm_spark.sql(f"INSERT INTO {table_fqn} VALUES (2, 'new')")
+
+    _assert_sail_describes_datasource_table(spark, table_fqn, provider=fmt)
+    properties = _describe_extended_properties(spark, table_fqn)
+    assert properties.get("Location") == new_location, properties
+    sail_rows = spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in sail_rows] == [(2, "new")]
+
+
+@pytest.mark.parametrize("fmt", _HMS_METADATA_FORMATS)
+def test_spark_dsv2_table_properties_round_trip_spark_sql_hidden(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Foreign TBLPROPERTIES survive; spark.sql.* internal keys are hidden by Sail.
+
+    Datasource tables auto-carry ``spark.sql.sources.provider`` and
+    ``spark.sql.sources.schema.*`` properties. Sail must expose user properties
+    (e.g. ``interop.owner``) while hiding these ``spark.sql.`` internals.
+    """
+    table_fqn = f"{hms_s3_database}.props_{fmt}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (id INT, name STRING)
+        USING {fmt.upper()}
+        TBLPROPERTIES ('interop.owner' = 'spark')
+        """
+    )
+
+    sail_properties = _describe_extended_properties(spark, table_fqn)
+    assert "interop.owner=spark" in sail_properties.get("Table Properties", ""), sail_properties
+    assert "spark.sql." not in sail_properties.get("Table Properties", ""), sail_properties
+
+
+@pytest.mark.parametrize("fmt", _HMS_METADATA_FORMATS)
+def test_spark_dsv2_column_comments_preserved(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Column comments set by Spark are preserved when Sail reads the table."""
+    table_fqn = f"{hms_s3_database}.comments_{fmt}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT COMMENT 'identifier',
+          name STRING
+        )
+        USING {fmt.upper()}
+        """
+    )
+
+    assert _describe_column_comments(spark, table_fqn)["id"] == "identifier"
+
+
+# ---------------------------------------------------------------------------
+# Partitioned tables.
+#
+# Sail's HMS catalog carries partition column *names* in TableStatus; actual
+# partition discovery is delegated to the object-store-aware reader, which walks
+# Hive-style partition directories (e.g. ``.../event_date=2024-01-02/...``).
+# These tests validate that path for plain (identity) partitions with simple
+# types, in isolation from the complex-type concerns that the existing xfailed
+# ``schema_matrix`` / ``mixed_complex_partitioned`` parquet tests confound.
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("fmt", _DSV2_FORMATS)
+def test_spark_creates_sail_reads_partitioned(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Spark writes an identity-partitioned table; Sail discovers partitions and reads.
+
+    ``PARTITIONED BY (event_date DATE)`` with simple types, isolated from complex
+    types. Covers: Sail preserves the partition column in schema, the reader
+    discovers Hive-style partition directories, partition pruning works, and values
+    (including the partition column) are restored correctly.
+    """
+    table_fqn = f"{hms_s3_database}.part_{fmt}"
+
+    jvm_spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT,
+          name STRING,
+          event_date DATE
+        )
+        USING {fmt.upper()}
+        PARTITIONED BY (event_date)
+        """
+    )
+    jvm_spark.sql(
+        f"""
+        INSERT INTO {table_fqn} VALUES
+          (1, 'alice', DATE '2024-01-02'),
+          (2, 'bob', DATE '2024-01-03'),
+          (3, 'cara', DATE '2024-01-02')
+        """
+    )
+
+    # Reference sanity check.
+    ref_rows = jvm_spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert len(ref_rows) == 3, f"Reference Spark expected 3 rows, got {len(ref_rows)}"
+
+    # Sail reads all rows across both partitions.
+    sail_rows = spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert [(r.id, r.name) for r in sail_rows] == [(1, "alice"), (2, "bob"), (3, "cara")]
+
+    # Partition pruning: Sail pushes the partition filter down (only one partition read).
+    pruned = spark.sql(
+        f"""
+        SELECT id, name, CAST(event_date AS STRING) AS event_date
+        FROM {table_fqn}
+        WHERE event_date = DATE '2024-01-02'
+        ORDER BY id
+        """
+    ).collect()
+    assert [(r.id, r.name, r.event_date) for r in pruned] == [
+        (1, "alice", "2024-01-02"),
+        (3, "cara", "2024-01-02"),
+    ]
+
+
+@pytest.mark.parametrize("fmt", _SAIL_WRITE_FORMATS)
+def test_sail_creates_spark_reads_partitioned(
+    spark: SparkSession,
+    jvm_spark: SparkSession,
+    hms_s3_database: str,
+    fmt: str,
+) -> None:
+    """Sail writes an identity-partitioned table; Spark discovers partitions and reads.
+
+    Reverse direction: Sail's ``build_generic_table`` records the partition columns in
+    HMS; Sail's writer lays out Hive-style partition directories; reference Spark
+    reads them back through HMS.
+    """
+    table_fqn = f"{hms_s3_database}.reverse_part_{fmt}"
+
+    spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT,
+          name STRING,
+          event_date DATE
+        )
+        USING {fmt.upper()}
+        PARTITIONED BY (event_date)
+        """
+    )
+    spark.sql(
+        f"""
+        INSERT INTO {table_fqn} VALUES
+          (1, 'alice', DATE '2024-01-02'),
+          (2, 'bob', DATE '2024-01-03')
+        """
+    )
+
+    # Assert the partition column (event_date) round-trips in the all-rows path,
+    # not just the data columns. Catches a writer that drops the partition column
+    # from the schema or writes wrong partition values.
+    sail_rows = spark.sql(
+        f"""
+        SELECT id, name, CAST(event_date AS STRING) AS event_date
+        FROM {table_fqn}
+        ORDER BY id
+        """
+    ).collect()
+    assert [(r.id, r.name, r.event_date) for r in sail_rows] == [
+        (1, "alice", "2024-01-02"),
+        (2, "bob", "2024-01-03"),
+    ]
+
+    # Independent cross-engine validation: JVM Spark reads HMS metadata directly.
+    _assert_jvm_spark_sees_datasource_provider(jvm_spark, hms_s3_database, f"reverse_part_{fmt}", provider=fmt)
+
+    # Reference Spark reads Sail's partitioned layout through HMS, including the
+    # partition column values.
+    ref_rows = jvm_spark.sql(
+        f"""
+        SELECT id, name, CAST(event_date AS STRING) AS event_date
+        FROM {table_fqn}
+        ORDER BY id
+        """
+    ).collect()
+    assert [(r.id, r.name, r.event_date) for r in ref_rows] == [
+        (1, "alice", "2024-01-02"),
+        (2, "bob", "2024-01-03"),
+    ]
diff --git a/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_roundtrip.py b/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_roundtrip.py
index 56c09c2ff..526c221e1 100644
--- a/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_roundtrip.py
+++ b/python/pysail/tests/spark/catalog/hms/test_spark_to_sail_roundtrip.py
@@ -268,6 +268,30 @@ def test_spark_alters_datasource_table_location_sail_reads_new_path(
     assert [(r.id, r.name) for r in sail_rows] == [(2, "new")]
 
 
+def test_spark_creates_sail_reads_delta_datasource_table(
+    jvm_spark: SparkSession,
+    spark: SparkSession,
+    hms_s3_database: str,
+) -> None:
+    """Spark registers a Delta table in HMS as a datasource table; Sail reads it.
+
+    Delta Lake tables registered in the Hive Metastore are persisted as Spark
+    datasource tables carrying ``spark.sql.sources.provider=delta``. Sail must
+    recognise the provider and resolve the table location. This covers the Delta
+    branch of ``sail_catalog_hms::convert::table_location`` end-to-end.
+    """
+    table_fqn = f"{hms_s3_database}.roundtrip_delta_datasource"
+
+    jvm_spark.sql(f"CREATE TABLE {table_fqn} USING DELTA AS SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob'")
+
+    properties = _describe_extended_properties(spark, table_fqn)
+    assert properties.get("Provider", "").lower() == "delta"
+    assert properties.get("Location"), "Sail did not resolve a location for the Delta table"
+
+    sail_rows = spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+    assert [(row.id, row.name) for row in sail_rows] == [(1, "alice"), (2, "bob")]
+
+
 def test_spark_catalog_api_creates_table_sail_reads_external_table(
     jvm_spark: SparkSession,
     spark: SparkSession,
diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py b/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py
index 5e9da1ec8..3ad930412 100644
--- a/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py
+++ b/python/pysail/tests/spark/catalog/iceberg_rest/conftest.py
@@ -10,8 +10,10 @@
 from typing import TYPE_CHECKING
 
 import pytest
+import requests
 from testcontainers.core.container import DockerContainer
 from testcontainers.core.network import Network
+from testcontainers.core.wait_strategies import LogMessageWaitStrategy
 from testcontainers.core.waiting_utils import wait_for_logs
 
 from pysail.testing.spark.session import spark_connect_server
@@ -20,6 +22,9 @@
     from collections.abc import Generator
     from pathlib import Path
 
+LAKEKEEPER_IMAGE = "quay.io/lakekeeper/catalog:v0.12.1"
+LAKEKEEPER_DATABASE_URL = "postgresql://postgres:postgres@lakekeeper-db:5432/postgres"
+LAKEKEEPER_PROJECT_ID = "00000000-0000-0000-0000-000000000000"
 NESSIE_NAMESPACE_SEPARATOR = "-"
 
 
@@ -54,9 +59,9 @@ def seaweedfs_container(
         .with_exposed_ports(8333)
         .with_network(docker_network)
         .with_network_aliases("seaweedfs")
+        .waiting_for(LogMessageWaitStrategy("Start Seaweed S3 API").with_startup_timeout(120))
     )
     container.start()
-    wait_for_logs(container, "Start Seaweed S3 API", timeout=120)
     yield container
     container.stop()
 
@@ -102,6 +107,143 @@ def _create_s3_bucket(seaweedfs_host_endpoint: str) -> None:
             return
 
 
+def lakekeeper_command_container(
+    docker_network: Network,
+    command: str,
+) -> DockerContainer:
+    """Configure a Lakekeeper container for the test PostgreSQL database."""
+    return (
+        DockerContainer(LAKEKEEPER_IMAGE)
+        .with_command([command])
+        .with_env("LAKEKEEPER__PG_ENCRYPTION_KEY", "This-is-NOT-Secure!")
+        .with_env("LAKEKEEPER__PG_DATABASE_URL_READ", LAKEKEEPER_DATABASE_URL)
+        .with_env("LAKEKEEPER__PG_DATABASE_URL_WRITE", LAKEKEEPER_DATABASE_URL)
+        .with_network(docker_network)
+    )
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_database_container(
+    docker_network: Network,
+) -> Generator[DockerContainer, None, None]:
+    """Start the PostgreSQL database used by Lakekeeper."""
+    container = (
+        DockerContainer("postgres:17")
+        .with_env("POSTGRES_PASSWORD", "postgres")
+        .with_network(docker_network)
+        .with_network_aliases("lakekeeper-db")
+        .waiting_for(LogMessageWaitStrategy("database system is ready to accept connections").with_startup_timeout(120))
+    )
+    container.start()
+    try:
+        yield container
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="module")
+def _lakekeeper_migration(
+    docker_network: Network,
+    lakekeeper_database_container: DockerContainer,  # noqa: ARG001
+) -> None:
+    """Apply Lakekeeper database migrations before starting the server."""
+    container = lakekeeper_command_container(docker_network, "migrate")
+    container.start()
+    try:
+        result = container.get_wrapped_container().wait(timeout=120)
+        if result["StatusCode"] != 0:
+            stdout, stderr = container.get_logs()
+            message = (stdout + stderr).decode(errors="replace")
+            msg = f"Lakekeeper database migration failed:\n{message}"
+            raise RuntimeError(msg)
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_container(
+    docker_network: Network,
+    seaweedfs_container: DockerContainer,  # noqa: ARG001
+    _lakekeeper_migration: None,
+) -> Generator[DockerContainer, None, None]:
+    """Start Lakekeeper after its database has been migrated."""
+    container = (
+        lakekeeper_command_container(docker_network, "serve")
+        .with_exposed_ports(8181)
+        .with_network_aliases("lakekeeper")
+        .waiting_for(LogMessageWaitStrategy("Starting server on 0.0.0.0:8181").with_startup_timeout(120))
+    )
+    container.start()
+    try:
+        yield container
+    finally:
+        container.stop()
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_endpoint(lakekeeper_container: DockerContainer) -> str:
+    """Return a healthy host-accessible Lakekeeper endpoint."""
+    host = lakekeeper_container.get_container_host_ip()
+    port = lakekeeper_container.get_exposed_port(8181)
+    endpoint = f"http://{host}:{port}"
+
+    for attempt in range(30):
+        try:
+            response = requests.get(f"{endpoint}/health", timeout=10)
+            response.raise_for_status()
+        except requests.RequestException:
+            if attempt == 29:  # noqa: PLR2004
+                raise
+            time.sleep(1)
+        else:
+            return endpoint
+    msg = "unreachable"
+    raise AssertionError(msg)
+
+
+@pytest.fixture(scope="module")
+def lakekeeper_warehouse_id(
+    lakekeeper_endpoint: str,
+    seaweedfs_internal_endpoint: str,
+    _create_s3_bucket: None,
+) -> str:
+    """Bootstrap Lakekeeper and create the S3-compatible test warehouse."""
+    bootstrap = requests.post(
+        f"{lakekeeper_endpoint}/management/v1/bootstrap",
+        json={"accept-terms-of-use": True},
+        timeout=30,
+    )
+    bootstrap.raise_for_status()
+
+    warehouse = requests.post(
+        f"{lakekeeper_endpoint}/management/v1/warehouse",
+        json={
+            "warehouse-name": "demo",
+            "project-id": LAKEKEEPER_PROJECT_ID,
+            "storage-profile": {
+                "type": "s3",
+                "bucket": "icebergdata",
+                "key-prefix": "lakekeeper",
+                "endpoint": seaweedfs_internal_endpoint,
+                "region": "us-east-1",
+                "path-style-access": True,
+                "flavor": "s3-compat",
+                "sts-enabled": False,
+                "remote-signing-enabled": True,
+            },
+            "storage-credential": {
+                "type": "s3",
+                "credential-type": "access-key",
+                "access-key-id": "admin",
+                "secret-access-key": "password",
+            },
+        },
+        timeout=30,
+    )
+    warehouse.raise_for_status()
+    return warehouse.json()["warehouse-id"]
+
+
 @pytest.fixture(scope="module")
 def iceberg_rest_container(
     docker_network: Network,
diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py b/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py
index 2cf1d7b40..aabcd33cb 100644
--- a/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py
+++ b/python/pysail/tests/spark/catalog/iceberg_rest/test_commit.py
@@ -7,7 +7,9 @@
 from pathlib import PurePosixPath
 from typing import TYPE_CHECKING
 
+import boto3
 import pytest
+from botocore.config import Config
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -16,6 +18,7 @@
 
 
 NAMESPACE = "iceberg_commit_test"
+UNPARTITIONED_LAST_PARTITION_ID = 999
 UUID_METADATA_FILE_PATTERN = re.compile(
     r"^\d{5}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.metadata\.json$"
 )
@@ -36,6 +39,22 @@ def _load_table(iceberg_rest_endpoint: str, table_name: str) -> dict:
         return json.load(response)
 
 
+def _s3_object_keys(endpoint: str, location: str) -> set[str]:
+    parsed = urllib.parse.urlparse(location)
+    assert parsed.scheme == "s3"
+    prefix = parsed.path.lstrip("/").rstrip("/") + "/"
+    client = boto3.client(
+        "s3",
+        endpoint_url=endpoint,
+        aws_access_key_id="admin",
+        aws_secret_access_key="password",  # noqa: S106
+        region_name="us-east-1",
+        config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
+    )
+    pages = client.get_paginator("list_objects_v2").paginate(Bucket=parsed.netloc, Prefix=prefix)
+    return {item["Key"] for page in pages for item in page.get("Contents", [])}
+
+
 def _assert_uuid_metadata_location(metadata_location: str, expected_version: int | None = None) -> None:
     filename = PurePosixPath(metadata_location).name
     assert UUID_METADATA_FILE_PATTERN.match(filename), filename
@@ -49,6 +68,27 @@ def _current_schema_field_names(metadata: dict) -> list[str]:
     return [field["name"] for field in current_schema["fields"]]
 
 
+def _current_snapshot(metadata: dict) -> dict:
+    current_snapshot_id = metadata["current-snapshot-id"]
+    return next(snapshot for snapshot in metadata["snapshots"] if snapshot["snapshot-id"] == current_snapshot_id)
+
+
+def _assert_row_level_commit_metadata(
+    metadata: dict,
+    *,
+    previous_metadata_location: str,
+    operation: str,
+) -> dict:
+    snapshot = _current_snapshot(metadata)
+    assert snapshot["summary"]["operation"] == operation
+    assert metadata["last-sequence-number"] == snapshot["sequence-number"]
+    assert metadata["last-partition-id"] == UNPARTITIONED_LAST_PARTITION_ID
+    assert metadata["refs"]["main"]["snapshot-id"] == snapshot["snapshot-id"]
+    assert metadata["snapshot-log"][-1]["snapshot-id"] == snapshot["snapshot-id"]
+    assert metadata["metadata-log"][-1]["metadata-file"] == previous_metadata_location
+    return snapshot
+
+
 def test_ctas_records_rest_catalog_metadata_location(
     spark: SparkSession,
     iceberg_rest_endpoint: str,
@@ -121,6 +161,47 @@ def test_insert_advances_rest_catalog_metadata_location(
     assert [(row["id"], row["name"]) for row in rows] == [(1, "a"), (2, "b"), (3, "c")]
 
 
+def test_rest_catalog_write_honors_absolute_data_path(
+    spark: SparkSession,
+    iceberg_rest_endpoint: str,
+    seaweedfs_host_endpoint: str,
+) -> None:
+    table_name = "absolute_data_path_t"
+    table_fqn = f"{NAMESPACE}.{table_name}"
+    data_location = f"s3://icebergdata/managed-data/{NAMESPACE}/{table_name}"
+    spark.sql(f"DROP TABLE IF EXISTS {table_fqn}")
+    spark.sql(
+        f"""
+        CREATE TABLE {table_fqn} (
+          id INT,
+          name STRING
+        )
+        USING iceberg
+        TBLPROPERTIES ('write.data.path' = '{data_location}')
+        """
+    )
+
+    created = _load_table(iceberg_rest_endpoint, table_name)
+    assert created["metadata"]["properties"]["write.data.path"] == data_location
+
+    spark.sql(f"INSERT INTO {table_fqn} VALUES (1, 'one')")  # noqa: S608
+
+    committed = _load_table(iceberg_rest_endpoint, table_name)
+    assert committed["metadata-location"] != created["metadata-location"]
+    assert committed["metadata"]["current-snapshot-id"] not in (None, -1)
+    table_location = committed["metadata"]["location"]
+    assert table_location != data_location
+    assert committed["metadata"]["properties"]["write.data.path"] == data_location
+
+    data_keys = _s3_object_keys(seaweedfs_host_endpoint, data_location)
+    assert any(key.endswith(".parquet") for key in data_keys)
+    table_keys = _s3_object_keys(seaweedfs_host_endpoint, table_location)
+    assert not any(key.endswith(".parquet") for key in table_keys)
+
+    rows = spark.sql(f"SELECT id, name FROM {table_fqn}").collect()  # noqa: S608
+    assert [(row.id, row.name) for row in rows] == [(1, "one")]
+
+
 def test_merge_schema_append_advances_rest_catalog_metadata_location(
     spark: SparkSession,
     iceberg_rest_endpoint: str,
@@ -196,6 +277,160 @@ def test_insert_overwrite_advances_rest_catalog_metadata_location(
     assert [(row["id"], row["name"]) for row in rows] == [(3, "new"), (4, "new")]
 
 
+def test_delete_advances_rest_catalog_metadata_location_with_equality_delete(
+    spark: SparkSession,
+    iceberg_rest_endpoint: str,
+) -> None:
+    table_name = "delete_t"
+    spark.sql("DROP TABLE IF EXISTS iceberg_commit_test.delete_t")
+    spark.sql(
+        """
+        CREATE TABLE iceberg_commit_test.delete_t (
+          id INT,
+          name STRING,
+          flag STRING
+        )
+        USING iceberg
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.delete.mode' = 'merge-on-read'
+        )
+        """
+    )
+    spark.sql(
+        """
+        INSERT INTO iceberg_commit_test.delete_t
+        SELECT * FROM VALUES
+          (1, 'keep-a', 'keep'),
+          (2, 'drop-b', 'drop'),
+          (3, 'keep-c', 'keep')
+        """
+    )
+    before = _load_table(iceberg_rest_endpoint, table_name)
+    before_location = before["metadata-location"]
+    _assert_uuid_metadata_location(before_location, 1)
+
+    spark.sql("DELETE FROM iceberg_commit_test.delete_t WHERE flag = 'drop'")
+
+    after = _load_table(iceberg_rest_endpoint, table_name)
+    after_location = after["metadata-location"]
+    assert after_location != before_location
+    _assert_uuid_metadata_location(after_location, 2)
+    snapshot = _assert_row_level_commit_metadata(
+        after["metadata"],
+        previous_metadata_location=before_location,
+        operation="delete",
+    )
+    summary = snapshot["summary"]
+    assert summary["added-delete-files"] == "1"
+    assert summary["added-equality-delete-files"] == "1"
+    assert summary["added-equality-deletes"] == "1"
+    assert "deleted-records" not in summary
+    assert "added-position-delete-files" not in summary
+    assert summary["total-data-files"] == "1"
+    assert summary["total-delete-files"] == "1"
+    assert summary["total-records"] == "3"
+
+    rows = spark.sql("SELECT id, name, flag FROM iceberg_commit_test.delete_t ORDER BY id").collect()
+    assert [(row["id"], row["name"], row["flag"]) for row in rows] == [
+        (1, "keep-a", "keep"),
+        (3, "keep-c", "keep"),
+    ]
+
+
+def test_merge_advances_rest_catalog_metadata_location_with_position_delete(
+    spark: SparkSession,
+    iceberg_rest_endpoint: str,
+) -> None:
+    table_name = "merge_t"
+    spark.sql("DROP TABLE IF EXISTS iceberg_commit_test.merge_t")
+    spark.sql(
+        """
+        CREATE TABLE iceberg_commit_test.merge_t (
+          id INT,
+          name STRING,
+          flag STRING
+        )
+        USING iceberg
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+    )
+    spark.sql(
+        """
+        INSERT INTO iceberg_commit_test.merge_t
+        SELECT * FROM VALUES
+          (1, 'keep-a', 'keep'),
+          (2, 'old-b', 'update'),
+          (3, 'drop-c', 'delete'),
+          (5, 'old-e', 'expire'),
+          (6, 'drop-f', 'purge')
+        """
+    )
+    spark.sql(
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_rest_merge_source AS
+        SELECT * FROM VALUES
+          (2, 'new-b', 'insert'),
+          (3, 'ignored-c', 'delete'),
+          (4, 'new-d', 'insert')
+        AS src(id, name, flag)
+        """
+    )
+    before = _load_table(iceberg_rest_endpoint, table_name)
+    before_location = before["metadata-location"]
+    _assert_uuid_metadata_location(before_location, 1)
+
+    spark.sql(
+        """
+        MERGE INTO iceberg_commit_test.merge_t AS t
+        USING iceberg_rest_merge_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND t.flag = 'update' THEN
+          UPDATE SET name = s.name
+        WHEN MATCHED AND t.flag = 'delete' THEN
+          DELETE
+        WHEN NOT MATCHED THEN
+          INSERT (id, name, flag) VALUES (s.id, s.name, s.flag)
+        WHEN NOT MATCHED BY SOURCE AND t.flag = 'expire' THEN
+          UPDATE SET name = 'expired-e'
+        WHEN NOT MATCHED BY SOURCE AND t.flag = 'purge' THEN
+          DELETE
+        """
+    )
+
+    after = _load_table(iceberg_rest_endpoint, table_name)
+    after_location = after["metadata-location"]
+    assert after_location != before_location
+    _assert_uuid_metadata_location(after_location, 2)
+    snapshot = _assert_row_level_commit_metadata(
+        after["metadata"],
+        previous_metadata_location=before_location,
+        operation="overwrite",
+    )
+    summary = snapshot["summary"]
+    assert summary["added-delete-files"] == "2"
+    assert summary["added-position-delete-files"] == "2"
+    assert summary["added-position-deletes"] == "4"
+    assert "deleted-records" not in summary
+    assert summary["added-data-files"] == "2"
+    assert summary["added-records"] == "3"
+    assert summary["total-data-files"] == "3"
+    assert summary["total-delete-files"] == "2"
+    assert summary["total-position-deletes"] == "4"
+    assert summary["total-records"] == "8"
+
+    rows = spark.sql("SELECT id, name, flag FROM iceberg_commit_test.merge_t ORDER BY id").collect()
+    assert [(row["id"], row["name"], row["flag"]) for row in rows] == [
+        (1, "keep-a", "keep"),
+        (2, "new-b", "update"),
+        (4, "new-d", "insert"),
+        (5, "expired-e", "expire"),
+    ]
+
+
 def test_rest_catalog_rejects_catalog_managed_iceberg_alter(
     spark: SparkSession,
     iceberg_rest_endpoint: str,
diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py b/python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py
new file mode 100644
index 000000000..c8e107315
--- /dev/null
+++ b/python/pysail/tests/spark/catalog/iceberg_rest/test_lakekeeper.py
@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import urllib.parse
+from typing import TYPE_CHECKING
+
+import pytest
+import requests
+
+from pysail.testing.spark.session import spark_connect_server
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from pyspark.sql import SparkSession
+
+NAMESPACE = "lakekeeper_access_session_test"
+TABLE = f"sail.{NAMESPACE}.remote_signing_t"
+
+
+@pytest.fixture(scope="module")
+def remote(
+    lakekeeper_endpoint: str,
+    lakekeeper_warehouse_id: str,
+    seaweedfs_host_endpoint: str,
+) -> Generator[str, None, None]:
+    """Start Sail with the Lakekeeper-backed Iceberg REST catalog."""
+    del lakekeeper_warehouse_id
+    catalog_config = f'[{{name="sail", type="iceberg-rest", uri="{lakekeeper_endpoint}/catalog", warehouse="demo"}}]'
+    with spark_connect_server(
+        envs={
+            "SAIL_CATALOG__LIST": catalog_config,
+            "AWS_ACCESS_KEY_ID": "admin",
+            "AWS_SECRET_ACCESS_KEY": "password",
+            "AWS_REGION": "us-east-1",
+            "AWS_ENDPOINT": seaweedfs_host_endpoint,
+            "AWS_VIRTUAL_HOSTED_STYLE_REQUEST": "false",
+            "AWS_ALLOW_HTTP": "true",
+        },
+    ) as server:
+        yield server.remote
+
+
+@pytest.fixture(scope="module", autouse=True)
+def namespace(spark: SparkSession) -> Generator[None, None, None]:
+    spark.sql(f"CREATE NAMESPACE IF NOT EXISTS sail.{NAMESPACE}")
+    yield
+    spark.sql(f"DROP NAMESPACE IF EXISTS sail.{NAMESPACE} CASCADE")
+
+
+def _load_lakekeeper_table(
+    lakekeeper_endpoint: str,
+    lakekeeper_warehouse_id: str,
+) -> dict:
+    namespace = urllib.parse.quote(NAMESPACE, safe="")
+    table = urllib.parse.quote(TABLE.rsplit(".", 1)[1], safe="")
+    response = requests.get(
+        f"{lakekeeper_endpoint}/catalog/v1/{lakekeeper_warehouse_id}/namespaces/{namespace}/tables/{table}",
+        timeout=30,
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def test_create_write_uses_configured_credentials_with_lakekeeper_session_hints(
+    spark: SparkSession,
+    lakekeeper_endpoint: str,
+    lakekeeper_warehouse_id: str,
+) -> None:
+    spark.sql(f"DROP TABLE IF EXISTS {TABLE}")
+    source = spark.createDataFrame([(1, "a"), (2, "b")], ["id", "name"])
+    source.writeTo(TABLE).using("iceberg").create()
+
+    rows = spark.table(TABLE).orderBy("id").collect()
+    assert [(row["id"], row["name"]) for row in rows] == [(1, "a"), (2, "b")]
+
+    table = _load_lakekeeper_table(lakekeeper_endpoint, lakekeeper_warehouse_id)
+    assert table["config"]["s3.remote-signing-enabled"] == "true"
+    assert table["storage-credentials"]
diff --git a/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py b/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py
index 8e3650a5b..c38040c60 100644
--- a/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py
+++ b/python/pysail/tests/spark/catalog/iceberg_rest/test_nessie.py
@@ -7,6 +7,7 @@
 import pytest
 
 from pysail.testing.spark.session import spark_connect_server
+from pysail.tests.spark.catalog.iceberg_rest.test_commit import _s3_object_keys
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -15,10 +16,23 @@
 
 
 @pytest.fixture(scope="module")
-def remote(nessie_iceberg_rest_endpoint: str) -> Generator[str, None, None]:
+def remote(
+    nessie_iceberg_rest_endpoint: str,
+    seaweedfs_host_endpoint: str,
+) -> Generator[str, None, None]:
     """Start Sail server with Nessie as the Iceberg REST catalog."""
     catalog_config = f'[{{name="sail", type="iceberg-rest", uri="{nessie_iceberg_rest_endpoint}"}}]'
-    with spark_connect_server(envs={"SAIL_CATALOG__LIST": catalog_config}) as server:
+    with spark_connect_server(
+        envs={
+            "SAIL_CATALOG__LIST": catalog_config,
+            "AWS_ACCESS_KEY_ID": "admin",
+            "AWS_SECRET_ACCESS_KEY": "password",
+            "AWS_REGION": "us-east-1",
+            "AWS_ENDPOINT": seaweedfs_host_endpoint,
+            "AWS_VIRTUAL_HOSTED_STYLE_REQUEST": "false",
+            "AWS_ALLOW_HTTP": "true",
+        },
+    ) as server:
         yield server.remote
 
 
@@ -39,3 +53,34 @@ def test_create_table_with_nessie_catalog(spark: SparkSession) -> None:
         assert rows[0].isTemporary is False
     finally:
         spark.sql(f"DROP DATABASE IF EXISTS {namespace} CASCADE").collect()
+
+
+def test_nessie_write_honors_absolute_data_path(
+    spark: SparkSession,
+    seaweedfs_host_endpoint: str,
+) -> None:
+    namespace = "nessie_absolute_data_path"
+    table = f"{namespace}.t1"
+    data_location = f"s3://icebergdata/managed-data/{namespace}/t1"
+    warehouse_namespace = f"s3://icebergdata/nessie/{namespace}"
+
+    try:
+        spark.sql(f"CREATE DATABASE IF NOT EXISTS {namespace}").collect()
+        spark.sql(
+            f"""
+            CREATE TABLE {table} (id INT, name STRING)
+            USING iceberg
+            TBLPROPERTIES ('write.data.path' = '{data_location}')
+            """
+        ).collect()
+        spark.sql(f"INSERT INTO {table} VALUES (1, 'one')")  # noqa: S608
+
+        data_keys = _s3_object_keys(seaweedfs_host_endpoint, data_location)
+        assert any(key.endswith(".parquet") for key in data_keys)
+        warehouse_keys = _s3_object_keys(seaweedfs_host_endpoint, warehouse_namespace)
+        assert not any(key.endswith(".parquet") for key in warehouse_keys)
+
+        rows = spark.sql(f"SELECT id, name FROM {table}").collect()  # noqa: S608
+        assert [(row.id, row.name) for row in rows] == [(1, "one")]
+    finally:
+        spark.sql(f"DROP DATABASE IF EXISTS {namespace} CASCADE").collect()
diff --git a/python/pysail/tests/spark/catalog/test_function.py b/python/pysail/tests/spark/catalog/test_function.py
new file mode 100644
index 000000000..7561a6310
--- /dev/null
+++ b/python/pysail/tests/spark/catalog/test_function.py
@@ -0,0 +1,226 @@
+"""Tests for Spark Catalog function APIs."""
+
+import pytest
+
+SPARK_4_2_VERSION_UPDATE_BUILT_INS = {
+    "current_path",
+    "is_valid_variant",
+    "kll_merge_agg_bigint",
+    "kll_merge_agg_double",
+    "kll_merge_agg_float",
+    "measure",
+    "time_bucket",
+    "time_from_micros",
+    "time_from_millis",
+    "time_from_seconds",
+    "time_to_micros",
+    "time_to_millis",
+    "time_to_seconds",
+    "tuple_difference_double",
+    "tuple_difference_integer",
+    "tuple_difference_theta_double",
+    "tuple_difference_theta_integer",
+    "tuple_intersection_agg_double",
+    "tuple_intersection_agg_integer",
+    "tuple_intersection_double",
+    "tuple_intersection_integer",
+    "tuple_intersection_theta_double",
+    "tuple_intersection_theta_integer",
+    "tuple_sketch_agg_double",
+    "tuple_sketch_agg_integer",
+    "tuple_sketch_estimate_double",
+    "tuple_sketch_estimate_integer",
+    "tuple_sketch_summary_double",
+    "tuple_sketch_summary_integer",
+    "tuple_sketch_theta_double",
+    "tuple_sketch_theta_integer",
+    "tuple_union_agg_double",
+    "tuple_union_agg_integer",
+    "tuple_union_double",
+    "tuple_union_integer",
+    "tuple_union_theta_double",
+    "tuple_union_theta_integer",
+    "vector_avg",
+    "vector_cosine_similarity",
+    "vector_inner_product",
+    "vector_l2_distance",
+    "vector_norm",
+    "vector_normalize",
+    "vector_sum",
+}
+
+
+def _show_function_names(spark, sql):
+    return {row.function for row in spark.sql(sql).collect()}
+
+
+def test_list_functions_returns_spark_function_fields(spark):
+    functions = spark.catalog.listFunctions()
+    assert functions
+
+    function = next(f for f in functions if f.name == "to_date")
+    assert function._fields == (
+        "name",
+        "catalog",
+        "namespace",
+        "description",
+        "className",
+        "isTemporary",
+    )
+    assert function.description.startswith("\n    to_date(date_str[, fmt]) - ")
+    assert "`date_str`" in function.description
+    assert function.className == "org.apache.spark.sql.catalyst.expressions.ParseToDate"
+    assert function.isTemporary is True
+
+
+def test_list_functions_includes_built_ins(spark):
+    names = {f.name for f in spark.catalog.listFunctions()}
+    assert {
+        "+",
+        "<>",
+        "between",
+        "current_database",
+        "to_date",
+        "window",
+        "||",
+    }.issubset(names)
+
+
+def test_list_functions_includes_spark_4_2_version_update_built_ins(spark):
+    names = {f.name for f in spark.catalog.listFunctions()}
+    assert SPARK_4_2_VERSION_UPDATE_BUILT_INS.issubset(names)
+
+
+def test_list_functions_has_metadata_for_built_ins(spark):
+    functions = {f.name: f for f in spark.catalog.listFunctions()}
+    expected = {
+        "+": "org.apache.spark.sql.catalyst.expressions.Add",
+        "concat": "org.apache.spark.sql.catalyst.expressions.Concat",
+        "current_path": "org.apache.spark.sql.catalyst.expressions.CurrentPath",
+        "time_bucket": "org.apache.spark.sql.catalyst.expressions.TimeBucketExpressionBuilder",
+        "to_date": "org.apache.spark.sql.catalyst.expressions.ParseToDate",
+        "tuple_difference_double": "org.apache.spark.sql.catalyst.expressions.TupleDifferenceDouble",
+        "vector_norm": "org.apache.spark.sql.catalyst.expressions.VectorNorm",
+    }
+    for name, class_name in expected.items():
+        assert functions[name].isTemporary is True
+        assert functions[name].className == class_name
+        assert functions[name].description
+
+
+def test_list_functions_includes_user_registered_udfs(spark):
+    function_name = "catalog_add_one"
+    spark.sql(f"DROP TEMPORARY FUNCTION IF EXISTS {function_name}")
+    try:
+        spark.udf.register(
+            function_name,
+            lambda value: value + 1 if value is not None else None,
+            "long",
+        )
+
+        functions = {f.name: f for f in spark.catalog.listFunctions()}
+        function = functions[function_name]
+        assert function.name == function_name
+        assert function.isTemporary is True
+        assert function_name in {f.name for f in spark.catalog.listFunctions("default", function_name)}
+        assert _show_function_names(spark, f"SHOW USER FUNCTIONS LIKE '{function_name}'") == {function_name}
+        assert _show_function_names(spark, f"SHOW SYSTEM FUNCTIONS LIKE '{function_name}'") == set()
+        assert _show_function_names(spark, f"SHOW ALL FUNCTIONS LIKE '{function_name}'") == {function_name}
+        assert spark.sql(f"SELECT {function_name}(1) AS value").collect()[0].value == 2  # noqa PLR2004
+
+        rows = [row.function_desc for row in spark.sql(f"DESCRIBE FUNCTION {function_name}").collect()]
+        assert rows[0] == f"Function: {function_name}"
+        assert rows[-1] == "Usage: N/A."
+    finally:
+        spark.sql(f"DROP TEMPORARY FUNCTION IF EXISTS {function_name}")
+
+
+def test_list_functions_respects_database_and_pattern(spark):
+    names = {f.name for f in spark.catalog.listFunctions("default", "to*")}
+    assert "to_date" in names
+    assert "to_timestamp" in names
+    assert "+" not in names
+    assert "current_database" not in names
+
+
+def test_show_functions_returns_spark_sql_shape(spark):
+    functions = spark.sql("SHOW FUNCTIONS")
+    assert functions.columns == ["function"]
+
+    names = {row.function for row in functions.where("function IN ('+', 'to_date')").collect()}
+    assert names == {"+", "to_date"}
+
+
+def test_describe_function_returns_signature_and_description(spark):
+    result = spark.sql("DESCRIBE FUNCTION to_date")
+    assert result.columns == ["function_desc"]
+
+    rows = [row.function_desc for row in result.collect()]
+    expected_usage = (
+        "Usage: \n"
+        "    to_date(date_str[, fmt]) - Parses the `date_str` expression with the `fmt` expression to\n"
+        "      a date. Returns null with invalid input. By default, it follows casting rules to a date if\n"
+        "      the `fmt` is omitted.\n"
+        "  "
+    )
+    assert rows == [
+        "Function: to_date",
+        "Class: org.apache.spark.sql.catalyst.expressions.ParseToDate",
+        expected_usage,
+    ]
+
+
+def test_describe_function_extended_adds_extended_usage(spark):
+    rows = [row.function_desc for row in spark.sql("DESC FUNCTION EXTENDED to_date").collect()]
+    assert rows[-1].startswith("Extended Usage:\n    Arguments:\n")
+    assert "* date_str - A string to be parsed to date." in rows[-1]
+    assert "\n    Examples:\n" in rows[-1]
+    assert "> SELECT to_date('2009-07-30 04:17:52');" in rows[-1]
+    assert "    Since: 1.5.0" in rows[-1]
+
+
+def test_describe_function_supports_string_literal_name(spark):
+    rows = [row.function_desc for row in spark.sql("DESC FUNCTION 'concat'").collect()]
+    assert rows[0] == "Function: concat"
+    assert rows[1] == "Class: org.apache.spark.sql.catalyst.expressions.Concat"
+    assert rows[2].startswith("Usage: concat(col1, col2, ..., colN) - ")
+
+
+def test_describe_function_supports_operator_name(spark):
+    rows = [row.function_desc for row in spark.sql("DESCRIBE FUNCTION +").collect()]
+    assert rows[0] == "Function: +"
+    assert rows[1] == "Class: org.apache.spark.sql.catalyst.expressions.Add"
+    assert rows[2].startswith("Usage: expr1 + expr2 - ")
+
+
+def test_describe_function_reports_unknown_function(spark):
+    with pytest.raises(Exception, match=r"(?i)(not found|function)"):
+        spark.sql("DESCRIBE FUNCTION no_such_function").collect()
+    with pytest.raises(Exception, match=r"(?i)(not found|function)"):
+        spark.sql("DESCRIBE FUNCTION default.to_date").collect()
+
+
+def test_show_functions_respects_scope(spark):
+    assert _show_function_names(spark, "SHOW SYSTEM FUNCTIONS LIKE 'to_date'") == {"to_date"}
+    assert _show_function_names(spark, "SHOW USER FUNCTIONS LIKE 'to_date'") == set()
+    assert _show_function_names(spark, "SHOW ALL FUNCTIONS LIKE 'to_date'") == {"to_date"}
+
+
+def test_show_functions_respects_namespace_and_pattern(spark):
+    assert _show_function_names(spark, "SHOW FUNCTIONS IN default LIKE 'to_date'") == {"to_date"}
+    assert _show_function_names(spark, "SHOW FUNCTIONS FROM default LIKE 'to_date'") == {"to_date"}
+
+
+def test_show_functions_supports_legacy_identifier_pattern(spark):
+    assert _show_function_names(spark, "SHOW FUNCTIONS to_date") == {"to_date"}
+    assert _show_function_names(spark, "SHOW SYSTEM FUNCTIONS to_date") == {"to_date"}
+    assert _show_function_names(spark, "SHOW FUNCTIONS LIKE to_date") == {"to_date"}
+    assert _show_function_names(spark, "SHOW FUNCTIONS default.to_date") == {"to_date"}
+    assert _show_function_names(spark, "SHOW FUNCTIONS no_such_db.to_date") == {"to_date"}
+
+
+def test_show_functions_requires_like_after_namespace(spark):
+    with pytest.raises(Exception, match=r"(?i)(expected|parse|syntax|extra input)"):
+        spark.sql("SHOW FUNCTIONS IN default to_date").collect()
+    with pytest.raises(Exception, match=r"(?i)(expected|parse|syntax|extra input)"):
+        spark.sql("SHOW FUNCTIONS IN default LIKE to_date").collect()
diff --git a/python/pysail/tests/spark/catalog/unity/test_managed_delta_dataframe.py b/python/pysail/tests/spark/catalog/unity/test_managed_delta_dataframe.py
index f303d9fa2..9035f3b0d 100644
--- a/python/pysail/tests/spark/catalog/unity/test_managed_delta_dataframe.py
+++ b/python/pysail/tests/spark/catalog/unity/test_managed_delta_dataframe.py
@@ -6,6 +6,9 @@
 import json
 from typing import TYPE_CHECKING
 
+import pytest
+
+from pysail.testing.spark.utils.common import pyspark_version
 from pysail.tests.spark.catalog.unity.conftest import _unity_delta_commit_info, _unity_table_info
 
 if TYPE_CHECKING:
@@ -125,3 +128,50 @@ def test_managed_delta_create_and_schema_merge_write_spark_struct_field_type_jso
     finally:
         spark.sql(f"DROP TABLE IF EXISTS {table_fqn}")
         spark.sql(f"DROP SCHEMA IF EXISTS {database} CASCADE")
+
+
+@pytest.mark.skipif(
+    pyspark_version() < (4,),
+    reason="DataFrame arguments in spark.sql() require Spark 4+",
+)
+def test_spark_sql_merge_dataframe_source_is_unity_managed(
+    spark: SparkSession,
+    unity_rest_url: str,
+) -> None:
+    database = "unity_dataframe_sql_args_test"
+    table = "managed_delta_sql_args_merge_t"
+    table_fqn = f"{database}.{table}"
+
+    spark.sql(f"CREATE SCHEMA IF NOT EXISTS {database}")
+    try:
+        spark.sql(f"DROP TABLE IF EXISTS {table_fqn}")
+        spark.sql(
+            f"""
+            CREATE TABLE {table_fqn}
+            USING delta
+            AS SELECT * FROM VALUES
+              (1, 'old'),
+              (2, 'keep')
+            AS t(id, name)
+            """
+        )
+
+        source_df = spark.createDataFrame([(1, "new"), (3, "insert")], schema="id INT, name STRING")
+        spark.sql(
+            f"""
+            MERGE INTO {table_fqn} AS tgt
+            USING {{src_df}} AS src
+            ON tgt.id = src.id
+            WHEN MATCHED THEN UPDATE SET name = src.name
+            WHEN NOT MATCHED THEN INSERT *
+            """,
+            src_df=source_df,
+        )
+
+        rows = spark.sql(f"SELECT id, name FROM {table_fqn} ORDER BY id").collect()
+        assert [(row.id, row.name) for row in rows] == [(1, "new"), (2, "keep"), (3, "insert")]
+
+        assert _unity_delta_commit_info(unity_rest_url, table_fqn, 2)
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_fqn}")
+        spark.sql(f"DROP SCHEMA IF EXISTS {database} CASCADE")
diff --git a/python/pysail/tests/spark/conftest.py b/python/pysail/tests/spark/conftest.py
index dd0411b50..c30943794 100644
--- a/python/pysail/tests/spark/conftest.py
+++ b/python/pysail/tests/spark/conftest.py
@@ -34,6 +34,10 @@ def pytest_configure(config):
         "markers",
         "integration: mark test as requiring external services and deselected by default",
     )
+    config.addinivalue_line(
+        "markers",
+        "function(group): categorize a function BDD scenario",
+    )
     # Load all pytest-bdd step modules.
     config.pluginmanager.import_plugin("pysail.testing.spark.steps.files")
     config.pluginmanager.import_plugin("pysail.testing.spark.steps.sql")
@@ -155,7 +159,20 @@ class DoctestMarker:
 ]
 
 
+FUNCTION_TAG_PATTERN = re.compile(r"function\((.*)\)")
+FUNCTION_TAG_VALUES = ["nullability", "columnargs", "lambda", "sketch"]
+SPARK_TAG_PATTERN = re.compile(r"spark-(.*)")
+
+
 def pytest_bdd_apply_tag(tag: str, function):
+    if (m := FUNCTION_TAG_PATTERN.fullmatch(tag)) is not None:
+        group = m.group(1)
+        if group not in FUNCTION_TAG_VALUES:
+            msg = f"invalid function tag: {tag}"
+            raise ValueError(msg)
+        pytest.mark.function(group=group)(function)
+        return True
+
     if tag == "sail-only":
         pytest.mark.skipif(is_jvm_spark(), reason="Sail-only feature not supported by JVM Spark")(function)
         return True
@@ -165,13 +182,16 @@ def pytest_bdd_apply_tag(tag: str, function):
         pytest.mark.xfail(not is_jvm_spark(), reason="Known Sail bug", strict=True)(function)
         return True
 
-    if tag.startswith("spark-"):
-        match = re.fullmatch(r"spark-(\d+(?:\.\d+)*)", tag)
-        if match is None:
+    if (m := SPARK_TAG_PATTERN.fullmatch(tag)) is not None:
+        try:
+            s = m.group(1)
+            version = tuple(int(part) for part in s.split("."))
+            if not version:
+                msg = "empty version"
+                raise ValueError(msg)  # noqa: TRY301
+        except ValueError as e:
             msg = f"invalid Spark version tag: {tag}"
-            raise ValueError(msg)
-        s = match.group(1)
-        version = tuple(int(part) for part in s.split("."))
+            raise ValueError(msg) from e
         pytest.mark.skipif(pyspark_version() < version, reason=f"Requires Spark {s}+")(function)
         return True
 
diff --git a/python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml b/python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml
new file mode 100644
index 000000000..2caec5990
--- /dev/null
+++ b/python/pysail/tests/spark/dataframe/__snapshots__/test_checkpoint.plan.yaml
@@ -0,0 +1,59 @@
+# serializer version: 1
+---
+name: "test_checkpoint_preserves_partitioning_and_ordering[checkpoint]"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as key, #3@1 as count]
+      ProjectionExec: expr=[#1@0 as #2, count(Int32(1))@1 as #3]
+        AggregateExec: mode=SinglePartitioned, gby=[#1@0 as #1], aggr=[count(Int32(1))], ordering_mode=Sorted
+          ProjectionExec: expr=[#1@1 as #1]
+            DataSourceExec: file_groups={4 groups: }, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet
+---
+name: "test_checkpoint_preserves_partitioning_and_ordering[checkpoint].1"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#0@0 as id, #1@1 as key]
+      DataSourceExec: file_groups={4 groups: }, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet
+---
+name: "test_checkpoint_preserves_partitioning_and_ordering[local-checkpoint]"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as key, #3@1 as count]
+      ProjectionExec: expr=[#1@0 as #2, count(Int32(1))@1 as #3]
+        AggregateExec: mode=SinglePartitioned, gby=[#1@0 as #1], aggr=[count(Int32(1))], ordering_mode=Sorted
+          ProjectionExec: expr=[#1@1 as #1]
+            DataSourceExec: file_groups={4 groups: }, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet
+---
+name: "test_checkpoint_preserves_partitioning_and_ordering[local-checkpoint].1"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#0@0 as id, #1@1 as key]
+      DataSourceExec: file_groups={4 groups: }, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet
+---
+name: "test_checkpoint_supports_iterative_weakly_connected_components"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#24@0 as node, #25@1 as label]
+      ProjectionExec: expr=[#0@0 as #24, least(#1@1, CASE WHEN #20@2 IS NOT NULL THEN #20@2 ELSE #1@1 END) as #25]
+        HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#0@0, #19@0)], projection=[#0@0, #1@1, #20@3]
+          CoalescePartitionsExec
+            DataSourceExec: file_groups={4 groups: }, projection=[_c0@0 as #0, _c1@1 as #1], file_type=parquet
+          ProjectionExec: expr=[#4@0 as #19, min(#13)@1 as #20]
+            AggregateExec: mode=FinalPartitioned, gby=[#4@0 as #4], aggr=[min(#13)]
+              RepartitionExec: partitioning=Hash([#4@0], 4), input_partitions=4
+                AggregateExec: mode=Partial, gby=[#4@0 as #4], aggr=[min(#13)]
+                  RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
+                    HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#12@0, #5@1)], projection=[#4@2, #13@1]
+                      CoalescePartitionsExec
+                        ProjectionExec: expr=[#10@0 as #12, #11@1 as #13]
+                          DataSourceExec: file_groups={4 groups: }, projection=[_c0@0 as #10, _c1@1 as #11], file_type=parquet
+                      UnionExec
+                        ProjectionExec: expr=[a@0 as #4, b@1 as #5]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+                        ProjectionExec: expr=[b@1 as #4, a@0 as #5]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
diff --git a/python/pysail/tests/spark/dataframe/__snapshots__/test_repartition.plan.yaml b/python/pysail/tests/spark/dataframe/__snapshots__/test_repartition.plan.yaml
new file mode 100644
index 000000000..af08bb171
--- /dev/null
+++ b/python/pysail/tests/spark/dataframe/__snapshots__/test_repartition.plan.yaml
@@ -0,0 +1,49 @@
+# serializer version: 1
+---
+name: "test_explicit_repartition_pushes_column_projection_down_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as id1, #7@1 as id2]
+      ExplicitRepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=4
+        ProjectionExec: expr=[#1@0 as #6, #3@1 as #7]
+          HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #5@0)], projection=[#1@0, #3@1]
+            HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #3@0)]
+              RepartitionExec: partitioning=Hash([#1@0], 4), input_partitions=1
+                ProjectionExec: expr=[id@0 as #1]
+                  RangeExec
+              RepartitionExec: partitioning=Hash([#3@0], 4), input_partitions=1
+                ProjectionExec: expr=[id@0 as #3]
+                  RangeExec
+            RepartitionExec: partitioning=Hash([#5@0], 4), input_partitions=1
+              ProjectionExec: expr=[id@0 as #5]
+                RangeExec
+---
+name: "test_explicit_repartition_hash_partitioning_remaps_after_projection_pushdown_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as id1, #7@1 as id2]
+      RepartitionExec: partitioning=Hash([#6@0, #7@1], 3), input_partitions=4
+        ProjectionExec: expr=[#1@0 as #6, #3@1 as #7]
+          HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #5@0)], projection=[#1@0, #3@1]
+            HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #3@0)]
+              RepartitionExec: partitioning=Hash([#1@0], 4), input_partitions=1
+                ProjectionExec: expr=[id@0 as #1]
+                  RangeExec
+              RepartitionExec: partitioning=Hash([#3@0], 4), input_partitions=1
+                ProjectionExec: expr=[id@0 as #3]
+                  RangeExec
+            RepartitionExec: partitioning=Hash([#5@0], 4), input_partitions=1
+              ProjectionExec: expr=[id@0 as #5]
+                RangeExec
+---
+name: "test_explicit_repartition_does_not_push_filter_down_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#0@0 as id]
+      FilterExec: #0@0 % 2 = 0
+        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3
+          ExplicitRepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1
+            RangeExec
diff --git a/python/pysail/tests/spark/dataframe/test_checkpoint.py b/python/pysail/tests/spark/dataframe/test_checkpoint.py
new file mode 100644
index 000000000..5852f8b54
--- /dev/null
+++ b/python/pysail/tests/spark/dataframe/test_checkpoint.py
@@ -0,0 +1,169 @@
+import shutil
+
+import pytest
+from pyspark import StorageLevel
+from pyspark.sql import functions as sf
+
+from pysail.testing.spark.session import spark_connect_server
+from pysail.testing.spark.steps.plan import normalize_plan_text
+from pysail.testing.spark.utils.common import pyspark_version
+
+pytestmark = pytest.mark.skipif(
+    pyspark_version() < (4,),
+    reason="checkpoint and localCheckpoint require PySpark Connect 4+",
+)
+
+
+@pytest.fixture(scope="module")
+def checkpoint_root(tmp_path_factory):
+    root = tmp_path_factory.mktemp("object-store-checkpoints") / "__checkpoint_testing__"
+    root.mkdir()
+    return root
+
+
+@pytest.fixture(scope="module")
+def remote(checkpoint_root):
+    with spark_connect_server(envs={"SAIL_EXECUTION__CHECKPOINT__PATH": checkpoint_root.as_uri()}) as server:
+        yield server.remote
+
+
+@pytest.mark.parametrize("local", [False, True], ids=["checkpoint", "local-checkpoint"])
+def test_eager_checkpoint_is_an_object_store_snapshot(spark, checkpoint_root, tmp_path, local):
+    source_path = tmp_path / "source"
+    spark.createDataFrame([(1, "a"), (2, "b"), (3, "c")], "id INT, value STRING").write.parquet(str(source_path))
+    source = spark.read.parquet(str(source_path)).where("id <= 2")
+    files_before = {path for path in checkpoint_root.rglob("*") if path.is_file()}
+
+    checkpointed = source.localCheckpoint() if local else source.checkpoint()
+
+    checkpoint_files = {path for path in checkpoint_root.rglob("*") if path.is_file()} - files_before
+    assert checkpoint_files
+    assert all(path.suffix == ".parquet" for path in checkpoint_files)
+
+    shutil.rmtree(source_path)
+    assert checkpointed.orderBy("id").collect() == [(1, "a"), (2, "b")]
+    assert checkpointed.where("id = 2").select("value").collect() == [("b",)]
+
+
+def test_checkpoint_preserves_duplicate_column_names(spark):
+    checkpointed = (
+        spark.range(3, numPartitions=2)
+        .selectExpr(
+            "id AS value",
+            "id + 10 AS value",
+        )
+        .checkpoint()
+    )
+
+    assert [field.name for field in checkpointed.schema.fields] == ["value", "value"]
+    assert sorted(tuple(row) for row in checkpointed.collect()) == [
+        (0, 10),
+        (1, 11),
+        (2, 12),
+    ]
+
+
+def test_checkpoint_preserves_rows_with_no_columns(spark):
+    checkpointed = spark.range(3, numPartitions=2).select().checkpoint()
+
+    assert checkpointed.schema.simpleString() == "struct<>"
+    assert checkpointed.collect() == [(), (), ()]
+
+
+def test_checkpoint_preserves_field_metadata(spark):
+    checkpointed = (
+        spark.range(1).select(sf.col("id").alias("value", metadata={"source": "checkpoint-test"})).checkpoint()
+    )
+
+    assert checkpointed.schema["value"].metadata == {"source": "checkpoint-test"}
+
+
+def _weakly_connected_components(nodes, pairs):
+    forward = pairs.select(sf.col("a").alias("source"), sf.col("b").alias("target"))
+    reverse = pairs.select(sf.col("b").alias("source"), sf.col("a").alias("target"))
+    edges = forward.unionByName(reverse)
+    labels = nodes.select(sf.col("id").alias("node")).withColumn("label", sf.col("node"))
+
+    for _ in range(100):
+        neighbor_labels = labels.select(
+            sf.col("node").alias("target"),
+            sf.col("label").alias("target_label"),
+        )
+        neighbor_minimum = (
+            edges.join(neighbor_labels, on="target")
+            .groupBy("source")
+            .agg(sf.min("target_label").alias("neighbor_minimum"))
+            .select(sf.col("source").alias("node"), "neighbor_minimum")
+        )
+        next_labels = labels.join(neighbor_minimum, on="node", how="left").select(
+            "node",
+            sf.least(
+                "label",
+                sf.coalesce("neighbor_minimum", "label"),
+            ).alias("label"),
+        )
+        previous_labels = labels.select("node", sf.col("label").alias("previous_label"))
+        changed = (
+            next_labels.join(previous_labels, on="node")
+            .where(sf.col("label") != sf.col("previous_label"))
+            .limit(1)
+            .count()
+        )
+        labels = next_labels
+        if changed == 0:
+            break
+        labels = labels.checkpoint()
+    else:
+        pytest.fail("weakly connected components did not converge")
+
+    return labels
+
+
+def _component_partition(labels):
+    components = {}
+    for row in labels.collect():
+        components.setdefault(row.label, set()).add(row.node)
+    return {frozenset(members) for members in components.values()}
+
+
+@pytest.mark.yamlsnapshot(group="plan")
+def test_checkpoint_supports_iterative_weakly_connected_components(spark, snapshot):
+    nodes = spark.range(7)
+    pairs = spark.createDataFrame(
+        [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)],
+        "a LONG, b LONG",
+    )
+
+    expected = {frozenset(range(6)), frozenset({6})}
+    checkpointed = _weakly_connected_components(nodes, pairs)
+    checkpointed_plan = normalize_plan_text(checkpointed._explain_string())  # noqa: SLF001
+
+    assert _component_partition(checkpointed) == expected
+    assert checkpointed_plan == snapshot
+
+
+@pytest.mark.parametrize("local", [False, True], ids=["checkpoint", "local-checkpoint"])
+@pytest.mark.yamlsnapshot(group="plan")
+def test_checkpoint_preserves_partitioning_and_ordering(spark, local, snapshot):
+    source = (
+        spark.range(100, numPartitions=4)
+        .withColumn("key", sf.col("id") % 4)
+        .repartition(4, "key")
+        .sortWithinPartitions("key", "id")
+    )
+
+    checkpointed = source.localCheckpoint() if local else source.checkpoint()
+    aggregate_plan = normalize_plan_text(checkpointed.groupBy("key").count()._explain_string())  # noqa: SLF001
+    ordered_plan = normalize_plan_text(checkpointed.sortWithinPartitions("key", "id")._explain_string())  # noqa: SLF001
+
+    assert aggregate_plan == snapshot
+    assert ordered_plan == snapshot
+
+
+def test_checkpoint_rejects_unimplemented_fallback_semantics(spark):
+    source = spark.range(3)
+
+    with pytest.raises(Exception, match="lazy DataFrame checkpoint"):
+        source.checkpoint(eager=False)
+    with pytest.raises(Exception, match="StorageLevel"):
+        source.localCheckpoint(storageLevel=StorageLevel.DISK_ONLY)
diff --git a/python/pysail/tests/spark/dataframe/test_input_files.py b/python/pysail/tests/spark/dataframe/test_input_files.py
new file mode 100644
index 000000000..730d6377b
--- /dev/null
+++ b/python/pysail/tests/spark/dataframe/test_input_files.py
@@ -0,0 +1,127 @@
+"""Tests for the DataFrame ``inputFiles`` snapshot of constituent files."""
+
+import pytest
+from pyspark.sql.utils import AnalysisException
+
+
+def test_input_files_single_file(spark, tmp_path):
+    """Return the single file backing a one-partition dataset."""
+    spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}]).repartition(1).write.json(
+        str(tmp_path), mode="overwrite"
+    )
+    df = spark.read.format("json").load(str(tmp_path))
+
+    files = df.inputFiles()
+    assert len(files) == 1
+    assert all(isinstance(f, str) for f in files)
+    assert files[0].endswith(".json")
+
+
+def test_input_files_multiple_files(spark, tmp_path):
+    """Return one entry per file when the dataset spans multiple partitions."""
+    partitions = 3
+    spark.range(0, 10, 1, partitions).write.json(str(tmp_path), mode="overwrite")
+    df = spark.read.format("json").load(str(tmp_path))
+
+    assert len(df.inputFiles()) == partitions
+
+
+def test_input_files_union_is_deduplicated(spark, tmp_path):
+    """Files shared by both sides of a union are reported only once."""
+    partitions = 3
+    spark.range(0, 10, 1, partitions).write.json(str(tmp_path), mode="overwrite")
+    df = spark.read.format("json").load(str(tmp_path))
+
+    # The union references the same source twice; the files must not be double-counted.
+    assert len(df.union(df).inputFiles()) == partitions
+
+
+def test_input_files_excludes_hidden_markers(spark, tmp_path):
+    """Hidden files such as ``_SUCCESS`` and ``.crc`` are not part of the dataset."""
+    spark.range(3).repartition(1).write.json(str(tmp_path), mode="overwrite")
+    (tmp_path / "_SUCCESS").touch()
+    (tmp_path / ".crc").touch()
+    df = spark.read.format("json").load(str(tmp_path))
+
+    files = df.inputFiles()
+    assert len(files) == 1
+    assert all("_SUCCESS" not in f and not f.endswith("/.crc") for f in files)
+
+
+def test_input_files_are_percent_encoded(spark, tmp_path):
+    """Returned URIs percent-encode reserved characters such as spaces."""
+    path = str(tmp_path / "a b")
+    spark.range(1).repartition(1).write.json(path, mode="overwrite")
+    df = spark.read.format("json").load(path)
+
+    files = df.inputFiles()
+    assert len(files) == 1
+    assert "%20" in files[0]
+    assert " " not in files[0]
+
+
+def test_input_files_ignores_eliminated_scans(spark, tmp_path):
+    """A scan removed by optimization (``WHERE false``) contributes no files."""
+    spark.range(5).repartition(1).write.json(str(tmp_path), mode="overwrite")
+    df = spark.read.format("json").load(str(tmp_path))
+
+    assert df.filter("1 = 0").inputFiles() == []
+    assert len(df.inputFiles()) == 1
+
+
+def test_input_files_excludes_nested_hidden_directories(spark, tmp_path):
+    """A hidden directory nested below a visible one is excluded from reads and inputFiles."""
+    rows = 3
+    visible = tmp_path / "visible"
+    visible.mkdir()
+    (visible / "good.json").write_text("".join(f'{{"id": {i}}}\n' for i in range(rows)))
+    (visible / "_hidden").mkdir()
+    (visible / "_hidden" / "bad.json").write_text('{"id": 99}\n')
+    df = spark.read.format("json").load(str(tmp_path))
+
+    # The nested hidden file must be excluded from both the scan and inputFiles.
+    assert df.count() == rows
+    files = df.inputFiles()
+    assert len(files) == 1
+    assert all("_hidden" not in f for f in files)
+
+
+def test_input_files_excludes_explicitly_targeted_hidden_file(spark, tmp_path):
+    """An explicitly targeted hidden file is ignored, as in Spark."""
+    hidden = tmp_path / "_data.json"
+    hidden.write_text('{"id": 1}\n')
+    df = spark.read.schema("id long").json(str(hidden))
+
+    assert df.count() == 0
+    assert df.inputFiles() == []
+
+
+def test_input_files_honors_path_glob_filter(spark, tmp_path):
+    """A listing-format ``pathGlobFilter`` restricts the files that compose the dataset."""
+    (tmp_path / "keep.json").write_text('{"id": 1}\n')
+    (tmp_path / "drop.json").write_text('{"id": 2}\n')
+    df = spark.read.format("json").option("pathGlobFilter", "keep.*").load(str(tmp_path))
+
+    files = df.inputFiles()
+    assert len(files) == 1
+    assert files[0].endswith("keep.json")
+
+
+def test_input_files_matches_scan_under_glob_alternation(spark, tmp_path):
+    """``inputFiles`` and scans both support ``{a,b}`` alternation."""
+    (tmp_path / "a.png").write_bytes(b"\x89PNG")
+    (tmp_path / "b.jpg").write_bytes(b"\xff\xd8\xff")
+    (tmp_path / "c.pdf").write_bytes(b"%PDF")
+    df = spark.read.format("binaryFile").option("pathGlobFilter", "*.{png,jpg}").load(str(tmp_path))
+
+    assert df.count() == len(df.inputFiles())
+    assert {file.rsplit("/", 1)[-1] for file in df.inputFiles()} == {"a.png", "b.jpg"}
+
+
+def test_input_files_rejects_invalid_path_glob_filter(spark, tmp_path):
+    """A malformed ``pathGlobFilter`` surfaces as an error rather than being silently ignored."""
+    (tmp_path / "a.png").write_bytes(b"\x89PNG")
+    df = spark.read.format("binaryFile").option("pathGlobFilter", "[").load(str(tmp_path))
+
+    with pytest.raises(AnalysisException, match="path glob filter"):
+        df.inputFiles()
diff --git a/python/pysail/tests/spark/dataframe/test_pivot.py b/python/pysail/tests/spark/dataframe/test_pivot.py
index ecc13d839..dacd7d1e9 100644
--- a/python/pysail/tests/spark/dataframe/test_pivot.py
+++ b/python/pysail/tests/spark/dataframe/test_pivot.py
@@ -31,15 +31,29 @@ def test_pivot_infers_values_sorted(spark):
     assert_frame_equal(actual, expected)
 
 
+def test_pivot_infers_numeric_values_sorted_numerically(spark):
+    actual = (
+        spark.createDataFrame(
+            [("a", 10, 100), ("a", 2, 20), ("b", 2, 5)],
+            schema="g STRING, k INT, v INT",
+        )
+        .groupBy("g")
+        .pivot("k")
+        .sum("v")
+        .orderBy("g")
+        .toPandas()
+    )
+    assert list(actual.columns) == ["g", "2", "10"]
+    expected = pd.DataFrame({"g": ["a", "b"], "2": [20, 5], "10": [100.0, float("nan")]})
+    assert_frame_equal(actual, expected)
+
+
 def test_pivot_infers_float_values_with_nan_last(spark):
     # Edge case: pivoting on a floating-point column whose inferred values include NaN.
     # NaN is incomparable under `partial_cmp`, so the sort falls back to a deterministic
     # tiebreaker instead of treating NaN as equal to its neighbour (which left the column
     # order dependent on the non-deterministic aggregate row order). This pins that guarantee:
     # finite values come first in ascending order, with their `.0` preserved, and NaN sorts last.
-    #
-    # One unrelated Spark-parity gap is intentionally NOT asserted here: the `NaN` pivot column
-    # counts the `k=NaN` row whereas Spark yields NULL (see the dedicated xfail below).
     actual = (
         spark.createDataFrame(
             [
@@ -57,6 +71,7 @@ def test_pivot_infers_float_values_with_nan_last(spark):
         .toPandas()
     )
     assert list(actual.columns) == ["g", "1.0", "2.0", "NaN"]
+    assert actual["NaN"].isna().all()
     # Sanity: the first finite column (k=1.0) holds its sums for both groups.
     assert actual.set_index("g")["1.0"].to_dict() == {"a": 10, "b": 5}
 
@@ -97,6 +112,107 @@ def test_pivot_allows_duplicate_aliases(spark):
     assert actual.values.tolist() == [[2012, 20000, 10000], [2013, 30000, 48000]]
 
 
+def test_pivot_allows_literal_aggregate_expression(spark):
+    # A literal aggregate (Int32) goes through Spark's PivotFirst path, so an absent
+    # (group, value) combination is NULL, not the literal: group 'b' has no 'y' row, so
+    # b/y is NULL (JVM-verified). collect() keeps the exact NULL (toPandas would coerce
+    # the int column to float/NaN).
+    rows = spark.sql("""
+        SELECT * FROM (
+          SELECT g, k FROM VALUES
+            ('a', 'x'),
+            ('a', 'y'),
+            ('b', 'x')
+          AS t(g, k)
+        ) PIVOT (1 FOR (k) IN ('x', 'y'))
+        ORDER BY g
+    """)
+    assert rows.columns == ["g", "x", "y"]
+    assert [tuple(r) for r in rows.collect()] == [("a", 1, 1), ("b", 1, None)]
+
+
+def test_pivot_rejects_bare_column_outside_aggregate(spark):
+    with pytest.raises(
+        Exception,
+        match=r"Aggregate expression required for pivot, but 'v' did not appear in any aggregate function",
+    ):
+        spark.sql("""
+            SELECT * FROM (
+              SELECT g, k, v FROM VALUES
+                ('a', 'x', 10)
+              AS t(g, k, v)
+            ) PIVOT (count(*) + v FOR (k) IN ('x'))
+        """).collect()
+
+
+def test_pivot_first_last_string_ignores_nulls_like_spark_general_path(spark):
+    actual = spark.sql("""
+        SELECT * FROM (
+          SELECT g, k, s FROM VALUES
+            ('a', 'x', CAST(NULL AS STRING)),
+            ('a', 'x', 'alpha'),
+            ('b', 'x', CAST(NULL AS STRING)),
+            ('b', 'x', 'bravo')
+          AS t(g, k, s)
+        ) PIVOT (
+          first(s) AS first_s,
+          last(s) AS last_s
+          FOR (k) IN ('x')
+        )
+        ORDER BY g
+    """).toPandas()
+    assert list(actual.columns) == ["g", "x_first_s", "x_last_s"]
+    assert actual.values.tolist() == [["a", "alpha", "alpha"], ["b", "bravo", "bravo"]]
+
+
+def test_pivot_multi_column_struct(spark):
+    actual = spark.sql("""
+        SELECT * FROM (
+          SELECT year, course, training, earnings FROM VALUES
+            (2012, 'Java', 'Dummies', 20000),
+            (2012, 'dotNET', 'Experts', 10000),
+            (2013, 'Java', 'Dummies', 30000),
+            (2013, 'dotNET', 'Experts', 48000)
+          AS s(year, course, training, earnings)
+        ) PIVOT (sum(earnings) FOR (course, training)
+                 IN (('Java', 'Dummies'), ('dotNET', 'Experts')))
+        ORDER BY year
+    """).toPandas()
+    assert list(actual.columns) == ["year", "{Java, Dummies}", "{dotNET, Experts}"]
+    assert actual.values.tolist() == [[2012, 20000, 10000], [2013, 30000, 48000]]
+
+
+def test_pivot_multi_column_struct_alias(spark):
+    actual = spark.sql("""
+        SELECT * FROM (
+          SELECT year, course, training, earnings FROM VALUES
+            (2012, 'Java', 'Dummies', 20000),
+            (2012, 'dotNET', 'Experts', 10000),
+            (2013, 'Java', 'Dummies', 30000),
+            (2013, 'dotNET', 'Experts', 48000)
+          AS s(year, course, training, earnings)
+        ) PIVOT (sum(earnings) FOR (course, training)
+                 IN (('Java', 'Dummies') AS java_dummies))
+        ORDER BY year
+    """).toPandas()
+    assert list(actual.columns) == ["year", "java_dummies"]
+    assert actual.values.tolist() == [[2012, 20000], [2013, 30000]]
+
+
+def test_pivot_multi_column_value_arity_mismatch_errors(spark):
+    with pytest.raises(
+        Exception,
+        match="pivot value has 1 expressions, but pivot column has 2",
+    ):
+        spark.sql("""
+            SELECT * FROM (
+              SELECT year, course, training, earnings FROM VALUES
+                (2012, 'Java', 'Dummies', 20000)
+              AS s(year, course, training, earnings)
+            ) PIVOT (sum(earnings) FOR (course, training) IN ('Java'))
+        """).collect()
+
+
 # SQL for a pivot whose value alias collides with a grouping column, producing two
 # output columns both named `year` (Spark allows this). Shared by the pair of tests below.
 _PIVOT_GROUPING_COLLISION_SQL = """
@@ -119,17 +235,9 @@ def test_pivot_value_colliding_with_grouping_column(spark):
     assert rows == [(2012, 20000), (2013, 30000)]
 
 
-@pytest.mark.xfail(
-    strict=False,
-    reason="Known flaky bug: with duplicate output column names, the Arrow/toPandas path "
-    'intermittently fails with an internal field-id error (No field named s."#0") while '
-    "collect() always succeeds. Paired with the collect test above to isolate the failure "
-    "to toPandas. Separate core field-id resolution bug, not specific to pivot.",
-)
 def test_pivot_value_colliding_with_grouping_column_to_pandas(spark):
     # Same query as the collect-based test above, but materialized via toPandas. This is the
-    # path that intermittently breaks; the pairing pinpoints toPandas (Arrow conversion) as
-    # the culprit rather than the pivot resolution or execution itself.
+    # path that exercises duplicate output names through Arrow conversion
     actual = spark.sql(_PIVOT_GROUPING_COLLISION_SQL).toPandas()
     assert list(actual.columns) == ["year", "year"]
     assert sorted(actual.values.tolist()) == [[2012, 20000], [2013, 30000]]
@@ -140,7 +248,10 @@ def test_pivot_inference_rejects_too_many_distinct_values(spark):
     # errors beyond it. The distinct-value scan is bounded by a LIMIT of one past the cap,
     # so a high-cardinality pivot column never collects an unbounded result into the planner.
     df = spark.range(10001).selectExpr("CAST(0 AS INT) AS g", "id AS k", "CAST(1 AS INT) AS v")
-    with pytest.raises(Exception, match="10000"):
+    with pytest.raises(
+        Exception,
+        match=r"The pivot column k has more than 10000 distinct values.*spark\.sql\.pivotMaxValues",
+    ):
         df.groupBy("g").pivot("k").sum("v").collect()
 
 
@@ -154,7 +265,10 @@ def test_pivot_max_values_config_is_honored(spark):
             [("a", 1, 10), ("a", 2, 20), ("a", 3, 30)],
             schema="g STRING, k INT, v INT",
         )
-        with pytest.raises(Exception, match="more than 2 distinct values"):
+        with pytest.raises(
+            Exception,
+            match=r"The pivot column k has more than 2 distinct values.*spark\.sql\.pivotMaxValues",
+        ):
             df.groupBy("g").pivot("k").sum("v").collect()
     finally:
         spark.conf.set("spark.sql.pivotMaxValues", original)
@@ -195,14 +309,46 @@ def test_pivot_decimal_value_naming_matches_spark(spark):
     assert actual.values.tolist() == [["a", 10, 20], ["b", 5, 8]]
 
 
-@pytest.mark.xfail(
-    strict=True,
-    reason="A NaN pivot value matches the NaN row in Sail (the pivot predicate `k = NaN` "
-    "evaluates true, consistent with Sail's `=` operator), so the NaN column is populated. "
-    "Spark leaves it NULL: although Spark SQL's `NaN = NaN` is also true, its pivot uses raw "
-    "IEEE `==` in codegen (NaN != NaN) and so contradicts its own `=`. Matching that would make "
-    "Sail's pivot inconsistent with its own equality, so this extreme edge is left as-is.",
-)
+def test_pivot_value_cast_to_int_column(spark):
+    actual = spark.sql("""
+        SELECT * FROM (
+          SELECT g, k, v FROM VALUES
+            ('a', 1, 10), ('a', 2, 20), ('b', 1, 5)
+          AS t(g, k, v)
+        ) PIVOT (sum(v) FOR (k) IN (1.2))
+        ORDER BY g
+    """).toPandas()
+    assert list(actual.columns) == ["g", "1.2"]
+    assert actual.values.tolist() == [["a", 10], ["b", 5]]
+
+
+def test_pivot_value_cast_to_string_column(spark):
+    actual = spark.sql("""
+        SELECT * FROM (
+          SELECT g, k, v FROM VALUES
+            ('a', '1', 10), ('a', '01', 99),
+            ('a', '2', 20), ('a', '02', 99),
+            ('b', '1', 5),  ('b', '01', 99),
+            ('b', '2', 8),  ('b', '02', 99)
+          AS t(g, k, v)
+        ) PIVOT (sum(v) FOR (k) IN (1, 2))
+        ORDER BY g
+    """).toPandas()
+    assert list(actual.columns) == ["g", "1", "2"]
+    assert actual.values.tolist() == [["a", 10, 20], ["b", 5, 8]]
+
+
+def test_pivot_rejects_non_castable_value(spark):
+    with pytest.raises(Exception, match=r"(?i)cast"):
+        spark.sql("""
+            SELECT * FROM (
+              SELECT g, k, v FROM VALUES
+                ('a', 1, 10)
+              AS t(g, k, v)
+            ) PIVOT (sum(v) FOR (k) IN ('not-an-int'))
+        """).collect()
+
+
 def test_pivot_nan_value_yields_null_column_like_spark(spark):
     # In Spark the NaN row matches no pivot value, so the `NaN` column is entirely NULL.
     actual = (
@@ -290,3 +436,22 @@ def test_pivot_inference_includes_null_pivot_value(spark):
     )
     assert df.columns == ["g", "null", "1"]
     assert [tuple(r) for r in df.collect()] == [("a", 5, 10), ("b", None, 7)]
+
+
+def test_pivot_nan_value_matches_in_general_path(spark):
+    import pyspark.sql.functions as F  # noqa: N812
+
+    # Non-numeric aggregate -> Spark's general (If/EqualNullSafe) pivot path, where NaN DOES
+    # match (unlike the PivotFirst path). Pins the `spark_uses_pivot_first` gate's other branch.
+    actual = (
+        spark.createDataFrame(
+            [("a", 1.0, "x"), ("a", float("nan"), "z"), ("b", 1.0, "y")],
+            schema="g STRING, k DOUBLE, s STRING",
+        )
+        .groupBy("g")
+        .pivot("k")
+        .agg(F.first("s"))
+        .orderBy("g")
+        .toPandas()
+    )
+    assert actual.set_index("g")["NaN"].to_dict() == {"a": "z", "b": None}
diff --git a/python/pysail/tests/spark/dataframe/test_repartition.py b/python/pysail/tests/spark/dataframe/test_repartition.py
index d1a87be25..6c78fe5fa 100644
--- a/python/pysail/tests/spark/dataframe/test_repartition.py
+++ b/python/pysail/tests/spark/dataframe/test_repartition.py
@@ -93,6 +93,67 @@ def test_explicit_repartition_plan_shape_uses_expected_physical_nodes(spark):
     assert "RepartitionExec: partitioning=Hash([" in hash_plan
 
 
+@pytest.mark.skipif(is_jvm_spark(), reason="different plans in JVM Spark")
+@pytest.mark.yamlsnapshot(group="plan")
+def test_explicit_repartition_pushes_column_projection_down_plan(spark, snapshot):
+    df1 = spark.sql("SELECT id AS id1 FROM range(6)")
+    df2 = spark.sql("SELECT id AS id2 FROM range(6)")
+    df3 = spark.sql("SELECT id AS id3 FROM range(6)")
+    df = df1.join(df2, df1.id1 == df2.id2).join(df3, df1.id1 == df3.id3).repartition(3).select("id1", "id2")
+    plan = normalized_plan(df)
+
+    assert plan == snapshot
+
+
+def test_explicit_repartition_pushes_column_projection_down_result(spark):
+    df1 = spark.sql("SELECT id AS id1 FROM range(6)")
+    df2 = spark.sql("SELECT id AS id2 FROM range(6)")
+    df3 = spark.sql("SELECT id AS id3 FROM range(6)")
+    df = (
+        df1.join(df2, df1.id1 == df2.id2)
+        .join(df3, df1.id1 == df3.id3)
+        .repartition(3)
+        .select("id1", "id2")
+        .orderBy("id1", "id2")
+    )
+    result = df.collect()
+
+    assert [(row["id1"], row["id2"]) for row in result] == [
+        (0, 0),
+        (1, 1),
+        (2, 2),
+        (3, 3),
+        (4, 4),
+        (5, 5),
+    ]
+
+
+@pytest.mark.skipif(is_jvm_spark(), reason="different plans in JVM Spark")
+@pytest.mark.yamlsnapshot(group="plan")
+def test_explicit_repartition_hash_partitioning_remaps_after_projection_pushdown_plan(spark, snapshot):
+    df1 = spark.sql("SELECT id AS id1 FROM range(6)")
+    df2 = spark.sql("SELECT id AS id2 FROM range(6)")
+    df3 = spark.sql("SELECT id AS id3 FROM range(6)")
+    df = (
+        df1.join(df2, df1.id1 == df2.id2)
+        .join(df3, df1.id1 == df3.id3)
+        .repartition(3, "id1", "id2")
+        .select("id1", "id2")
+    )
+    plan = normalized_plan(df)
+
+    assert plan == snapshot
+
+
+@pytest.mark.skipif(is_jvm_spark(), reason="different plans in JVM Spark")
+@pytest.mark.yamlsnapshot(group="plan")
+def test_explicit_repartition_does_not_push_filter_down_plan(spark, snapshot):
+    df = spark.range(6).repartition(3).filter(F.col("id") % 2 == 0)
+    plan = normalized_plan(df)
+
+    assert plan == snapshot
+
+
 def test_explicit_coalesce(spark):
     assert partition_count(spark.range(0, 10, 1, 2).coalesce(1)) == 1
     assert partition_count(spark.range(0, 10, 1, 2).coalesce(2)) == 2  # noqa: PLR2004
diff --git a/python/pysail/tests/spark/dataframe/test_sql.py b/python/pysail/tests/spark/dataframe/test_sql.py
index 83e08cd03..ff9e07365 100644
--- a/python/pysail/tests/spark/dataframe/test_sql.py
+++ b/python/pysail/tests/spark/dataframe/test_sql.py
@@ -5,6 +5,15 @@ def test_default_can_be_column_name(spark):
     assert spark.sql("SELECT DEFAULT FROM VALUES (1) AS t(DEFAULT)").collect() == [(1,)]
 
 
+def test_sql_positional_parameters(spark):
+    assert spark.sql("SELECT * FROM range(10) WHERE id > ?", args=[7]).collect() == [(8,), (9,)]
+    assert spark.sql("SELECT ? AS v FROM range(10) WHERE id > ? ORDER BY id", args=[1, 7]).collect() == [
+        (1,),
+        (1,),
+    ]
+    assert spark.sql("SELECT ? AS v", args=[1, 2]).collect() == [(1,)]
+
+
 def test_keyword_as_explicit_column_alias(spark):
     # Keywords are not reserved in Spark and can be used as column aliases
     # when the `AS` keyword is explicit.
diff --git a/python/pysail/tests/spark/dataframe/test_udt.txt b/python/pysail/tests/spark/dataframe/test_udt.txt
index e0260434c..fa5fc99d0 100644
--- a/python/pysail/tests/spark/dataframe/test_udt.txt
+++ b/python/pysail/tests/spark/dataframe/test_udt.txt
@@ -1,5 +1,6 @@
 >>> from pyspark.ml.linalg import VectorUDT
->>> from pyspark.sql.types import StringType, LongType, StructType, UserDefinedType
+>>> from pyspark.sql.types import LongType, StructType
+>>> from pysail.tests.spark.dataframe.udt import NamedPythonUDT, UnnamedPythonUDT
 >>> schema = StructType().add("key", LongType()).add("vec", VectorUDT())
 >>> df = spark.createDataFrame(data=[], schema=schema)
 >>> df
@@ -8,14 +9,6 @@ DataFrame[key: bigint, vec: vector]
 root
  |-- key: long (nullable = true)
  |-- vec: vector (nullable = true)
->>> class UnnamedPythonUDT(UserDefinedType):
-...     @classmethod
-...     def sqlType(cls):
-...         return StringType()
-...     @classmethod
-...     def module(cls):
-...         return "__main__"
-...
 >>> schema = StructType().add("key", LongType()).add("a", UnnamedPythonUDT())
 >>> df = spark.createDataFrame(data=[], schema=schema)
 >>> df
@@ -24,10 +17,6 @@ DataFrame[key: bigint, a: udt]
 root
  |-- key: long (nullable = true)
  |-- a: pythonuserdefined (nullable = true)
->>> class NamedPythonUDT(UnnamedPythonUDT):
-...     def simpleString(cls):
-...         return "foo"
-...
 >>> schema = StructType().add("key", LongType()).add("b", NamedPythonUDT())
 >>> df = spark.createDataFrame(data=[], schema=schema)
 >>> df
diff --git a/python/pysail/tests/spark/dataframe/udt.py b/python/pysail/tests/spark/dataframe/udt.py
new file mode 100644
index 000000000..4f868a2b0
--- /dev/null
+++ b/python/pysail/tests/spark/dataframe/udt.py
@@ -0,0 +1,25 @@
+"""Importable Python UDTs used by doctests.
+
+Spark resolves Python UDTs by importing their declared module and no longer
+unpickles UDT classes embedded in schema metadata. These test UDTs therefore
+must be defined in a module instead of directly in the doctest namespace.
+
+Reference: 
+"""
+
+from pyspark.sql.types import StringType, UserDefinedType
+
+
+class UnnamedPythonUDT(UserDefinedType):
+    @classmethod
+    def sqlType(cls):  # noqa: N802
+        return StringType()
+
+    @classmethod
+    def module(cls):
+        return __name__
+
+
+class NamedPythonUDT(UnnamedPythonUDT):
+    def simpleString(self):  # noqa: N802
+        return "foo"
diff --git a/python/pysail/tests/spark/datasource/test_binary.py b/python/pysail/tests/spark/datasource/test_binary.py
index 162ee07ee..6023146f0 100644
--- a/python/pysail/tests/spark/datasource/test_binary.py
+++ b/python/pysail/tests/spark/datasource/test_binary.py
@@ -92,6 +92,10 @@ def test_binary_read_options(spark, tmp_path):
     expected_single_count = 2
     assert single_char_df.count() == expected_single_count
 
+    alternation_df = spark.read.format("binaryFile").option("pathGlobFilter", "*.{png,jpeg}").load(str(tmp_path))
+    expected_alternation_count = 3
+    assert alternation_df.count() == expected_alternation_count
+
     all_df = spark.read.format("binaryFile").load(str(tmp_path))
     expected_all_count = 7
     assert all_df.count() == expected_all_count
@@ -131,3 +135,24 @@ def test_binary_read_projections(spark, tmp_path):
     large_count = large_files.count()
     expected_large = sum(1 for _, content in files.values() if len(content) > min_file_size)
     assert large_count == expected_large
+
+
+def test_binary_read_partition_columns(spark, tmp_path):
+    root = tmp_path / "binary_partitioned"
+    png_dir = root / "kind=image" / "ext=png"
+    bin_dir = root / "kind=data" / "ext=bin"
+    png_dir.mkdir(parents=True)
+    bin_dir.mkdir(parents=True)
+    png_content = b"PNG partition payload"
+    bin_content = b"BIN partition payload"
+    (png_dir / "image.png").write_bytes(png_content)
+    (bin_dir / "payload.bin").write_bytes(bin_content)
+
+    read_df = spark.read.format("binaryFile").load(str(root)).select("path", "length", "content", "kind", "ext")
+
+    rows = {(row.path.split("/")[-1], row.length, bytes(row.content), row.kind, row.ext) for row in read_df.collect()}
+    assert rows == {
+        ("image.png", len(png_content), png_content, "image", "png"),
+        ("payload.bin", len(bin_content), bin_content, "data", "bin"),
+    }
+    assert read_df.filter("kind = 'image'").count() == 1
diff --git a/python/pysail/tests/spark/datasource/test_csv.py b/python/pysail/tests/spark/datasource/test_csv.py
index fa2360bf9..ac7bbf7e7 100644
--- a/python/pysail/tests/spark/datasource/test_csv.py
+++ b/python/pysail/tests/spark/datasource/test_csv.py
@@ -42,6 +42,17 @@ def test_csv_read_write_basic(spark, sample_df, tmp_path, infer_schema):
     assert sorted(read_df.collect(), key=safe_sort_key) == sorted(expected, key=safe_sort_key)
 
 
+def test_csv_path_glob_filter(spark, tmp_path):
+    path = tmp_path / "csv_path_glob_filter"
+    path.mkdir()
+    (path / "keep.csv").write_text("value\n1\n")
+    (path / "drop.csv").write_text("value\n2\n")
+
+    df = spark.read.option("header", True).option("inferSchema", True).option("pathGlobFilter", "keep.*").csv(str(path))
+
+    assert df.collect() == [Row(value=1)]
+
+
 @pytest.mark.parametrize("infer_schema", [True, False])
 def test_csv_read_write_compressed(spark, sample_df, sample_pandas_df, tmp_path, infer_schema):
     # Round-tripped values are typed under `inferSchema=True` and STRING-only
diff --git a/python/pysail/tests/spark/datasource/test_json.py b/python/pysail/tests/spark/datasource/test_json.py
index 3fe81faf9..36bea873d 100644
--- a/python/pysail/tests/spark/datasource/test_json.py
+++ b/python/pysail/tests/spark/datasource/test_json.py
@@ -20,6 +20,18 @@ def test_json_read_write_basic(spark, sample_df, tmp_path):
     assert sorted(sample_df.collect(), key=safe_sort_key) == sorted(read_df.collect(), key=safe_sort_key)
 
 
+def test_json_path_glob_filter_applies_before_schema_inference(spark, tmp_path):
+    path = tmp_path / "json_path_glob_filter"
+    path.mkdir()
+    (path / "keep.json").write_text('{"id": 1}\n')
+    (path / "drop.json").write_text('{"excluded": true}\n')
+
+    df = spark.read.option("pathGlobFilter", "keep.*").json(str(path))
+
+    assert df.columns == ["id"]
+    assert df.collect() == [Row(id=1)]
+
+
 def test_json_write_compression(spark, sample_df, tmp_path):
     """Test JSON write with compression."""
     path = str(tmp_path / "json_compressed")
diff --git a/python/pysail/tests/spark/datasource/test_parquet.py b/python/pysail/tests/spark/datasource/test_parquet.py
index ce77f1a02..7f762e694 100644
--- a/python/pysail/tests/spark/datasource/test_parquet.py
+++ b/python/pysail/tests/spark/datasource/test_parquet.py
@@ -24,6 +24,22 @@ def test_parquet_read_write_basic(spark, sample_df, tmp_path):
     assert sorted(sample_df.collect(), key=safe_sort_key) == sorted(read_df.collect(), key=safe_sort_key)
 
 
+def test_parquet_path_glob_filter(spark, tmp_path):
+    keep_source = tmp_path / "parquet_keep_source"
+    drop_source = tmp_path / "parquet_drop_source"
+    spark.createDataFrame([(1,)], "id INT").coalesce(1).write.parquet(str(keep_source))
+    spark.createDataFrame([(2,)], "id INT").coalesce(1).write.parquet(str(drop_source))
+
+    path = tmp_path / "parquet_path_glob_filter"
+    path.mkdir()
+    next(keep_source.glob("*.parquet")).rename(path / "keep.parquet")
+    next(drop_source.glob("*.parquet")).rename(path / "drop.parquet")
+
+    df = spark.read.option("pathGlobFilter", "keep.*").parquet(str(path))
+
+    assert df.collect() == [Row(id=1)]
+
+
 def test_parquet_write_modes(spark, tmp_path):
     path = str(tmp_path / "parquet_write_modes")
 
@@ -115,6 +131,42 @@ def test_parquet_read_options(spark, sample_df, tmp_path):
     assert sorted(sample_df.collect(), key=safe_sort_key) == sorted(read_df.collect(), key=safe_sort_key)
 
 
+@pytest.mark.parametrize(
+    ("physical_schema", "requested_schema", "value"),
+    [
+        ("value DOUBLE", "value INT", 1.5),
+        ("value INT", "value BOOLEAN", 1),
+        ("value STRUCT", "value STRUCT", Row(nested=1.5)),
+        ("value ARRAY", "value ARRAY", [1.5]),
+        ("value MAP", "value MAP", {"key": 1.5}),
+    ],
+)
+def test_parquet_explicit_schema_rejects_incompatible_types(spark, tmp_path, physical_schema, requested_schema, value):
+    path = str(tmp_path / "incompatible_explicit_schema")
+    spark.createDataFrame([(value,)], physical_schema).write.parquet(path)
+
+    with pytest.raises(Exception, match="PARQUET_COLUMN_DATA_TYPE_MISMATCH"):
+        spark.read.schema(requested_schema).parquet(path).collect()
+
+
+def test_parquet_explicit_schema_allows_supported_widening(spark, tmp_path):
+    path = str(tmp_path / "supported_explicit_schema_widening")
+    spark.createDataFrame([(1, 1.5), (None, None)], "id INT, value FLOAT").write.parquet(path)
+
+    rows = spark.read.schema("id BIGINT, value DOUBLE").parquet(path).orderBy("id").collect()
+
+    assert rows == [Row(id=None, value=None), Row(id=1, value=1.5)]
+
+
+def test_parquet_explicit_schema_allows_missing_fields(spark, tmp_path):
+    path = str(tmp_path / "missing_explicit_schema_field")
+    spark.createDataFrame([(1,)], "id INT").write.parquet(path)
+
+    rows = spark.read.schema("id INT, missing STRING").parquet(path).collect()
+
+    assert rows == [Row(id=1, missing=None)]
+
+
 def test_parquet_write_with_bloom_filter(spark, tmpdir):
     def size(p):
         return get_data_directory_size(p, extension=".parquet")
@@ -190,15 +242,12 @@ def test_parquet_format_path(spark, sample_df, tmp_path):
 
 
 def test_parquet_read_with_custom_extension(spark, sample_pandas_df, tmp_path):
-    """Parquet files written under a non-standard suffix (e.g. `.hive`, as
-    emitted by Hive-managed tables) can be read via the `extension` /
-    `fileExtension` option."""
+    """`pathGlobFilter` selects Parquet files with a custom suffix."""
     directory = tmp_path / "parquet_custom_extension"
     directory.mkdir()
     file_path = directory / "data.hive"
     sample_pandas_df.to_parquet(str(file_path))
-    # TODO: add a file with another extension and the file should be ignored
-    #   when the filtering logic is implemented properly
+    sample_pandas_df.to_parquet(str(directory / "ignored.parquet"))
 
     expected_count = len(sample_pandas_df)
     expected_rows = sorted(sample_pandas_df.to_dict(orient="records"), key=safe_sort_key)
@@ -206,32 +255,17 @@ def test_parquet_read_with_custom_extension(spark, sample_pandas_df, tmp_path):
     def actual_rows(df):
         return sorted(df.toPandas().to_dict(orient="records"), key=safe_sort_key)
 
-    # Option key `extension`, directory path.
-    read_df = spark.read.option("extension", ".hive").parquet(str(directory))
-    assert read_df.count() == expected_count
-    assert actual_rows(read_df) == expected_rows
-
-    # Camel-case alias `fileExtension`, single-file path.
-    read_df = spark.read.option("fileExtension", ".hive").parquet(str(file_path))
-    assert read_df.count() == expected_count
-    assert actual_rows(read_df) == expected_rows
-
-    # Empty string disables extension filtering entirely.
-    read_df = spark.read.option("extension", "").parquet(str(directory))
+    read_df = spark.read.option("pathGlobFilter", "*.hive").parquet(str(directory))
     assert read_df.count() == expected_count
     assert actual_rows(read_df) == expected_rows
 
-    # SQL CREATE TABLE with OPTIONS (fileExtension '.hive').
-    # Use a separate directory so DROP TABLE side effects don't affect other cases.
-    sql_directory = tmp_path / "parquet_custom_extension_sql"
-    sql_directory.mkdir()
-    sample_pandas_df.to_parquet(str(sql_directory / "data.hive"))
+    # SQL CREATE TABLE with OPTIONS (pathGlobFilter '*.hive').
     table_name = "parquet_custom_extension_table"
     try:
         spark.sql(
             f"CREATE TABLE {table_name} USING parquet "
-            f"OPTIONS (fileExtension '.hive') "
-            f"LOCATION '{escape_sql_string_literal(str(sql_directory))}'"
+            f"OPTIONS (pathGlobFilter '*.hive') "
+            f"LOCATION '{escape_sql_string_literal(str(directory))}'"
         )
         read_df = spark.sql(f"SELECT * FROM {table_name}")  # noqa: S608
         assert read_df.count() == expected_count
diff --git a/python/pysail/tests/spark/datasource/test_python.py b/python/pysail/tests/spark/datasource/test_python.py
index a9fd39021..43dc00a96 100644
--- a/python/pysail/tests/spark/datasource/test_python.py
+++ b/python/pysail/tests/spark/datasource/test_python.py
@@ -5,7 +5,9 @@
 including both Arrow RecordBatch and tuple-based paths.
 """
 
+import json
 from collections.abc import Iterator
+from pathlib import Path
 from typing import Any
 from uuid import uuid4
 
@@ -13,6 +15,8 @@
 import pytest
 
 from pysail.testing.spark.session import spark_session_factory
+from pysail.testing.spark.utils.common import pyspark_version
+from pysail.testing.spark.utils.sql import escape_sql_string_literal
 
 try:
     from pyspark.sql.datasource import (
@@ -31,6 +35,21 @@
 except ImportError:
     pytest.skip("Python DataSource API is not available in this Spark version", allow_module_level=True)
 
+
+def _read_test_state(path: str | Path) -> dict[str, Any]:
+    path = Path(path)
+    if not path.exists():
+        return {}
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text) if text else {}
+
+
+def _update_test_state(path: str, **updates: Any) -> None:
+    state = _read_test_state(path)
+    state.update(updates)
+    Path(path).write_text(json.dumps(state, default=str), encoding="utf-8")
+
+
 # ============================================================================
 # RangeDataSource - Example datasource that generates sequential integers
 # ============================================================================
@@ -428,7 +447,7 @@ def read(self, partition):  # noqa: ARG002
 def test_python_default_single_partition(spark):
     """Test that missing partitions() defaults to a single partition."""
     import pyarrow as pa
-    from pyspark.sql.datasource import DataSource, DataSourceReader
+    from pyspark.sql.datasource import DataSource, DataSourceReader, InputPartition
 
     class DefaultPartitionDataSource(DataSource):
         """DataSource that relies on the default partitions()."""
@@ -445,8 +464,12 @@ def reader(self, schema):  # noqa: ARG002
 
     class DefaultPartitionReader(DataSourceReader):
         def read(self, partition):
-            if partition is not None:
-                msg = f"Expected default partition to be None, got {partition!r}"
+            if pyspark_version() >= (4, 2):
+                is_default_partition = isinstance(partition, InputPartition) and partition.value is None
+            else:
+                is_default_partition = partition is None
+            if not is_default_partition:
+                msg = f"Expected default partition, got {partition!r}"
                 raise ValueError(msg)
             batch = pa.RecordBatch.from_pydict({"id": [1, 2, 3]}, schema=pa.schema([("id", pa.int32())]))
             yield batch
@@ -724,6 +747,14 @@ def test_python_ddl_schema_fallback(spark):
     assert rows[0].id == 0
 
 
+def test_python_datasource_options_are_case_insensitive(spark):
+    spark.dataSource.register(RangeDataSource)
+
+    rows = spark.read.format("range").option("END", "2").load().collect()
+
+    assert sorted(row.id for row in rows) == [0, 1]
+
+
 def test_python_filter_pushdown_comparison(spark):
     """Test that comparison filters (>, <, >=, <=) work correctly.
 
@@ -896,9 +927,10 @@ def _create_writable_test_datasource():
     class InMemoryWriter(DataSourceWriter):
         """Row-based writer for testing."""
 
-        def __init__(self, schema, overwrite):
+        def __init__(self, schema, overwrite, state_path):
             self.schema = schema
             self.overwrite = overwrite
+            self.state_path = state_path
             self.data = []
             self.committed = False
             self.aborted = False
@@ -915,7 +947,7 @@ def commit(self, messages):
             """Commit the write."""
             self.committed = True
             self.commit_messages = messages
-            WritableDataSource.row_commit_messages = messages
+            _update_test_state(self.state_path, row_commit_messages=messages)
 
         def abort(self, messages):
             """Abort the write."""
@@ -925,9 +957,10 @@ def abort(self, messages):
     class InMemoryArrowWriter(DataSourceArrowWriter):
         """Arrow-based writer for testing."""
 
-        def __init__(self, schema, overwrite):
+        def __init__(self, schema, overwrite, state_path):
             self.schema = schema
             self.overwrite = overwrite
+            self.state_path = state_path
             self.batches = []
             self.committed = False
             self.aborted = False
@@ -947,7 +980,7 @@ def commit(self, messages):
             """Commit the write."""
             self.committed = True
             self.commit_messages = messages
-            WritableDataSource.arrow_commit_messages = messages
+            _update_test_state(self.state_path, arrow_commit_messages=messages)
 
         def abort(self, messages):
             """Abort the write."""
@@ -957,12 +990,10 @@ def abort(self, messages):
     class WritableDataSource(DataSource):
         """DataSource that supports both reading and writing."""
 
-        row_commit_messages = None
-        arrow_commit_messages = None
-
         def __init__(self, options):
             self.options = options
             self.writer_type = options.get("writer_type", "row")
+            self.state_path = options["state_path"]
 
         @classmethod
         def name(cls):
@@ -988,14 +1019,15 @@ def read(self, partition):  # noqa: ARG002
         def writer(self, schema, overwrite):
             """Return writer based on configuration."""
             if self.writer_type == "arrow":
-                return InMemoryArrowWriter(schema, overwrite)
-            return InMemoryWriter(schema, overwrite)
+                return InMemoryArrowWriter(schema, overwrite, self.state_path)
+            return InMemoryWriter(schema, overwrite, self.state_path)
 
     return WritableDataSource, datasource_name
 
 
-def test_python_basic_write(spark):
+def test_python_basic_write(spark, tmp_path):
     """Test basic Row-based write."""
+    state_path = tmp_path / "basic_write_state.json"
     writable_test_ds, datasource_name = _create_writable_test_datasource()
     spark.dataSource.register(writable_test_ds)
 
@@ -1003,16 +1035,19 @@ def test_python_basic_write(spark):
     df = spark.createDataFrame([(1, "a"), (2, "b"), (3, "c")], ["id", "value"])
 
     # Write data
-    df.write.format(datasource_name).option("writer_type", "row").mode("append").save()
+    df.write.format(datasource_name).option("writer_type", "row").option("state_path", str(state_path)).mode(
+        "append"
+    ).save()
 
     # Verify commit was called
-    row_messages = writable_test_ds.row_commit_messages
+    row_messages = _read_test_state(state_path).get("row_commit_messages")
     assert row_messages is not None
     assert len(row_messages) > 0
 
 
-def test_python_arrow_write(spark):
+def test_python_arrow_write(spark, tmp_path):
     """Test Arrow-based write."""
+    state_path = tmp_path / "arrow_write_state.json"
     writable_test_ds, datasource_name = _create_writable_test_datasource()
     spark.dataSource.register(writable_test_ds)
 
@@ -1020,20 +1055,24 @@ def test_python_arrow_write(spark):
     df = spark.createDataFrame([(1, "x"), (2, "y")], ["id", "value"])
 
     # Write data using Arrow writer
-    df.write.format(datasource_name).option("writer_type", "arrow").mode("append").save()
+    df.write.format(datasource_name).option("writer_type", "arrow").option("state_path", str(state_path)).mode(
+        "append"
+    ).save()
 
     # Verify commit was called
-    arrow_messages = writable_test_ds.arrow_commit_messages
+    arrow_messages = _read_test_state(state_path).get("arrow_commit_messages")
     assert arrow_messages is not None
     assert len(arrow_messages) > 0
 
 
-def test_python_write_overwrite_mode(spark):
+def test_python_write_overwrite_mode(spark, tmp_path):
     """Test that overwrite mode is passed correctly."""
+    state_path = tmp_path / "overwrite_state.json"
 
     class OverwriteCheckWriter(DataSourceWriter):
-        def __init__(self, schema, overwrite):  # noqa: ARG002
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
             self.overwrite = overwrite
+            self.state_path = state_path
             self.data = []
 
         def write(self, iterator):
@@ -1043,9 +1082,7 @@ def write(self, iterator):
 
         def commit(self, messages):  # noqa: ARG002
             # Store overwrite flag for verification
-            OverwriteCheckWriter._last_overwrite = self.overwrite
-
-    OverwriteCheckWriter._last_overwrite = None  # noqa: SLF001
+            _update_test_state(self.state_path, last_overwrite=self.overwrite)
 
     class OverwriteCheckDataSource(DataSource):
         def __init__(self, options):
@@ -1059,22 +1096,24 @@ def schema(self):
             return "id INT"
 
         def writer(self, schema, overwrite):
-            return OverwriteCheckWriter(schema, overwrite)
+            return OverwriteCheckWriter(schema, overwrite, self.options["state_path"])
 
     spark.dataSource.register(OverwriteCheckDataSource)
 
     df = spark.createDataFrame([(1,), (2,)], ["id"])
-    df.write.format("overwrite_check").mode("overwrite").save()
+    df.write.format("overwrite_check").option("state_path", str(state_path)).mode("overwrite").save()
 
-    assert OverwriteCheckWriter._last_overwrite is True  # noqa: SLF001
+    assert _read_test_state(state_path).get("last_overwrite") is True
 
 
-def test_python_write_append_mode(spark):
+def test_python_write_append_mode(spark, tmp_path):
     """Test that append mode is passed correctly."""
+    state_path = tmp_path / "append_state.json"
 
     class AppendCheckWriter(DataSourceWriter):
-        def __init__(self, schema, overwrite):  # noqa: ARG002
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
             self.overwrite = overwrite
+            self.state_path = state_path
             self.data = []
 
         def write(self, iterator):
@@ -1083,9 +1122,7 @@ def write(self, iterator):
             return {}
 
         def commit(self, messages):  # noqa: ARG002
-            AppendCheckWriter._last_overwrite = self.overwrite
-
-    AppendCheckWriter._last_overwrite = None  # noqa: SLF001
+            _update_test_state(self.state_path, last_overwrite=self.overwrite)
 
     class AppendCheckDataSource(DataSource):
         def __init__(self, options):
@@ -1099,21 +1136,23 @@ def schema(self):
             return "id INT"
 
         def writer(self, schema, overwrite):
-            return AppendCheckWriter(schema, overwrite)
+            return AppendCheckWriter(schema, overwrite, self.options["state_path"])
 
     spark.dataSource.register(AppendCheckDataSource)
 
     df = spark.createDataFrame([(1,), (2,)], ["id"])
-    df.write.format("append_check").mode("append").save()
+    df.write.format("append_check").option("state_path", str(state_path)).mode("append").save()
 
-    assert AppendCheckWriter._last_overwrite is False  # noqa: SLF001
+    assert _read_test_state(state_path).get("last_overwrite") is False
 
 
-def test_python_write_commit(spark):
+def test_python_write_commit(spark, tmp_path):
     """Test that commit is called with all partition messages."""
+    state_path = tmp_path / "commit_state.json"
 
     class CommitTrackingWriter(DataSourceWriter):
-        def __init__(self, schema, overwrite):  # noqa: ARG002
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
+            self.state_path = state_path
             self.data = []
 
         def write(self, iterator):
@@ -1121,9 +1160,7 @@ def write(self, iterator):
             return {"count": count}
 
         def commit(self, messages):
-            CommitTrackingWriter._commit_messages = messages
-
-    CommitTrackingWriter._commit_messages = []  # noqa: SLF001
+            _update_test_state(self.state_path, commit_messages=messages)
 
     class CommitTrackingDataSource(DataSource):
         def __init__(self, options):
@@ -1137,20 +1174,22 @@ def schema(self):
             return "id INT"
 
         def writer(self, schema, overwrite):
-            return CommitTrackingWriter(schema, overwrite)
+            return CommitTrackingWriter(schema, overwrite, self.options["state_path"])
 
     spark.dataSource.register(CommitTrackingDataSource)
 
     df = spark.createDataFrame([(1,), (2,), (3,)], ["id"])
-    df.write.format("commit_tracking").save()
+    df.write.format("commit_tracking").option("state_path", str(state_path)).save()
 
     # Verify commit was called with messages
-    assert CommitTrackingWriter._commit_messages is not None  # noqa: SLF001
-    assert len(CommitTrackingWriter._commit_messages) > 0  # noqa: SLF001
+    commit_messages = _read_test_state(state_path).get("commit_messages")
+    assert commit_messages is not None
+    assert len(commit_messages) > 0
 
 
-def test_python_write_empty_dataframe(spark):
+def test_python_write_empty_dataframe(spark, tmp_path):
     """Test writing an empty DataFrame."""
+    state_path = tmp_path / "empty_write_state.json"
     writable_test_ds, datasource_name = _create_writable_test_datasource()
     spark.dataSource.register(writable_test_ds)
 
@@ -1158,10 +1197,10 @@ def test_python_write_empty_dataframe(spark):
     df = spark.createDataFrame([], "id INT, value STRING")
 
     # Write should succeed even with empty data
-    df.write.format(datasource_name).option("writer_type", "row").save()
+    df.write.format(datasource_name).option("writer_type", "row").option("state_path", str(state_path)).save()
 
     # Verify commit was called
-    row_messages = writable_test_ds.row_commit_messages
+    row_messages = _read_test_state(state_path).get("row_commit_messages")
     assert row_messages is not None
 
 
@@ -1203,15 +1242,13 @@ def read(self, partition):  # noqa: ARG002
         df.write.format("readonly").save()
 
 
-def test_python_write_abort_on_failure(spark):
+def test_python_write_abort_on_failure(spark, tmp_path):
     """Test that abort is called when writer.write() raises an exception."""
+    state_path = tmp_path / "abort_on_failure_state.json"
 
     class FailingWriter(DataSourceWriter):
-        _abort_called = False
-        _abort_messages = None
-
-        def __init__(self, schema, overwrite):
-            pass
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
+            self.state_path = state_path
 
         def write(self, iterator):  # noqa: ARG002
             msg = "Intentional write failure for testing"
@@ -1223,11 +1260,7 @@ def commit(self, messages):  # noqa: ARG002
             raise AssertionError(msg)
 
         def abort(self, messages):
-            FailingWriter._abort_called = True
-            FailingWriter._abort_messages = messages
-
-    FailingWriter._abort_called = False  # noqa: SLF001
-    FailingWriter._abort_messages = None  # noqa: SLF001
+            _update_test_state(self.state_path, abort_called=True, abort_messages=messages)
 
     class FailingWriteDataSource(DataSource):
         def __init__(self, options):
@@ -1241,7 +1274,7 @@ def schema(self):
             return "id INT"
 
         def writer(self, schema, overwrite):
-            return FailingWriter(schema, overwrite)
+            return FailingWriter(schema, overwrite, self.options["state_path"])
 
     spark.dataSource.register(FailingWriteDataSource)
 
@@ -1249,18 +1282,17 @@ def writer(self, schema, overwrite):
 
     # Write should fail
     with pytest.raises(Exception, match=r"(?i)write failure|error"):
-        df.write.format("failing_write").save()
+        df.write.format("failing_write").option("state_path", str(state_path)).save()
 
     # Verify abort was called
-    assert FailingWriter._abort_called, "abort() should have been called on write failure"  # noqa: SLF001
+    assert _read_test_state(state_path).get("abort_called"), "abort() should have been called on write failure"
 
 
-def test_python_write_save_path_passed(spark):
+def test_python_write_save_path_passed(spark, tmp_path):
     """Test that .save("/my/path") passes the path to DataSource via options["path"]."""
+    state_path = tmp_path / "save_path_state.json"
 
     class PathCheckWriter(DataSourceWriter):
-        _received_path = None
-
         def __init__(self, schema, overwrite):
             pass
 
@@ -1276,7 +1308,7 @@ class PathCheckDataSource(DataSource):
         def __init__(self, options):
             self.options = options
             # Capture the path option for test verification
-            PathCheckDataSource._received_options = dict(options)
+            _update_test_state(options["state_path"], received_options=dict(options))
 
         @classmethod
         def name(cls):
@@ -1288,29 +1320,24 @@ def schema(self):
         def writer(self, schema, overwrite):
             return PathCheckWriter(schema, overwrite)
 
-    PathCheckDataSource._received_options = {}  # noqa: SLF001
-
     spark.dataSource.register(PathCheckDataSource)
 
     df = spark.createDataFrame([(1,), (2,)], ["id"])
-    df.write.format("path_check").mode("append").save("/my/test/path")
+    df.write.format("path_check").option("state_path", str(state_path)).mode("append").save("/my/test/path")
 
     # Verify the path was passed through options
-    assert "path" in PathCheckDataSource._received_options, (  # noqa: SLF001
-        "Save path should be passed to DataSource via options['path']"
-    )
-    assert PathCheckDataSource._received_options["path"] == "/my/test/path"  # noqa: SLF001
+    received_options = _read_test_state(state_path).get("received_options", {})
+    assert "path" in received_options, "Save path should be passed to DataSource via options['path']"
+    assert received_options["path"] == "/my/test/path"
 
 
-def test_python_write_commit_failure_triggers_abort(spark):
+def test_python_write_commit_failure_triggers_abort(spark, tmp_path):
     """Test that abort is called when commit() raises an exception."""
+    state_path = tmp_path / "commit_failure_state.json"
 
     class CommitFailWriter(DataSourceWriter):
-        _abort_called = False
-        _abort_messages = None
-
-        def __init__(self, schema, overwrite):
-            pass
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
+            self.state_path = state_path
 
         def write(self, iterator):
             count = sum(1 for _ in iterator)
@@ -1321,11 +1348,7 @@ def commit(self, messages):  # noqa: ARG002
             raise RuntimeError(msg)
 
         def abort(self, messages):
-            CommitFailWriter._abort_called = True
-            CommitFailWriter._abort_messages = messages
-
-    CommitFailWriter._abort_called = False  # noqa: SLF001
-    CommitFailWriter._abort_messages = None  # noqa: SLF001
+            _update_test_state(self.state_path, abort_called=True, abort_messages=messages)
 
     class CommitFailDataSource(DataSource):
         def __init__(self, options):
@@ -1339,7 +1362,7 @@ def schema(self):
             return "id INT"
 
         def writer(self, schema, overwrite):
-            return CommitFailWriter(schema, overwrite)
+            return CommitFailWriter(schema, overwrite, self.options["state_path"])
 
     spark.dataSource.register(CommitFailDataSource)
 
@@ -1347,16 +1370,15 @@ def writer(self, schema, overwrite):
 
     # Write should fail because commit fails
     with pytest.raises(Exception, match=r"(?i)commit failure|error"):
-        df.write.format("commit_fail").save()
+        df.write.format("commit_fail").option("state_path", str(state_path)).save()
 
     # Verify abort was called after commit failure
-    assert CommitFailWriter._abort_called, (  # noqa: SLF001
-        "abort() should have been called after commit failure"
-    )
+    assert _read_test_state(state_path).get("abort_called"), "abort() should have been called after commit failure"
 
 
-def test_python_write_null_values(spark):
+def test_python_write_null_values(spark, tmp_path):
     """Test writing data that contains null values."""
+    state_path = tmp_path / "null_values_state.json"
     writable_test_ds, datasource_name = _create_writable_test_datasource()
     spark.dataSource.register(writable_test_ds)
 
@@ -1367,26 +1389,29 @@ def test_python_write_null_values(spark):
     )
 
     # Test row-based write with nulls
-    df.write.format(datasource_name).option("writer_type", "row").mode("append").save()
-    row_messages = writable_test_ds.row_commit_messages
+    df.write.format(datasource_name).option("writer_type", "row").option("state_path", str(state_path)).mode(
+        "append"
+    ).save()
+    row_messages = _read_test_state(state_path).get("row_commit_messages")
     assert row_messages is not None
     assert len(row_messages) > 0
 
     # Test Arrow-based write with nulls
-    df.write.format(datasource_name).option("writer_type", "arrow").mode("append").save()
-    arrow_messages = writable_test_ds.arrow_commit_messages
+    df.write.format(datasource_name).option("writer_type", "arrow").option("state_path", str(state_path)).mode(
+        "append"
+    ).save()
+    arrow_messages = _read_test_state(state_path).get("arrow_commit_messages")
     assert arrow_messages is not None
     assert len(arrow_messages) > 0
 
 
-def test_python_write_none_commit_message(spark):
+def test_python_write_none_commit_message(spark, tmp_path):
     """Test that writer returning None as commit message is handled correctly."""
+    state_path = tmp_path / "none_commit_message_state.json"
 
     class NoneMessageWriter(DataSourceWriter):
-        _commit_messages = None
-
-        def __init__(self, schema, overwrite):
-            pass
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
+            self.state_path = state_path
 
         def write(self, iterator):
             for _ in iterator:
@@ -1394,9 +1419,7 @@ def write(self, iterator):
             # Explicitly return None (no commit message)
 
         def commit(self, messages):
-            NoneMessageWriter._commit_messages = messages
-
-    NoneMessageWriter._commit_messages = None  # noqa: SLF001
+            _update_test_state(self.state_path, commit_messages=messages)
 
     class NoneMessageDataSource(DataSource):
         def __init__(self, options):
@@ -1410,16 +1433,17 @@ def schema(self):
             return "id INT"
 
         def writer(self, schema, overwrite):
-            return NoneMessageWriter(schema, overwrite)
+            return NoneMessageWriter(schema, overwrite, self.options["state_path"])
 
     spark.dataSource.register(NoneMessageDataSource)
 
     df = spark.createDataFrame([(1,), (2,)], ["id"])
-    df.write.format("none_message").save()
+    df.write.format("none_message").option("state_path", str(state_path)).save()
 
     # Verify commit was called — messages list should contain the None message
-    assert NoneMessageWriter._commit_messages is not None  # noqa: SLF001
-    assert len(NoneMessageWriter._commit_messages) > 0  # noqa: SLF001
+    commit_messages = _read_test_state(state_path).get("commit_messages")
+    assert commit_messages is not None
+    assert len(commit_messages) > 0
 
 
 # ============================================================================
@@ -1549,14 +1573,14 @@ def read(self, partition):  # noqa: ARG002
         spark.sql(f"DROP TABLE IF EXISTS {table_name}")
 
 
-def test_python_sql_insert_into_python_datasource(spark):
+def test_python_sql_insert_into_python_datasource(spark, tmp_path):
     """Test INSERT INTO on a Python data source table."""
     table_name = f"sql_insert_{uuid4().hex[:8]}"
+    state_path = tmp_path / "sql_insert_state.json"
 
     class SQLInsertWriter(DataSourceWriter):
-        _commit_messages = None
-
-        def __init__(self, schema, overwrite):  # noqa: ARG002
+        def __init__(self, schema, overwrite, state_path):  # noqa: ARG002
+            self.state_path = state_path
             self.data = []
 
         def write(self, iterator):
@@ -1565,9 +1589,7 @@ def write(self, iterator):
             return {"count": len(self.data)}
 
         def commit(self, messages):
-            SQLInsertWriter._commit_messages = messages
-
-    SQLInsertWriter._commit_messages = None  # noqa: SLF001
+            _update_test_state(self.state_path, commit_messages=messages)
 
     class SQLInsertDataSource(DataSource):
         @classmethod
@@ -1581,7 +1603,7 @@ def reader(self, schema) -> DataSourceReader:  # noqa: ARG002
             return SQLInsertReader()
 
         def writer(self, schema, overwrite):
-            return SQLInsertWriter(schema, overwrite)
+            return SQLInsertWriter(schema, overwrite, self.options["state_path"])
 
     class SQLInsertReader(DataSourceReader):
         def partitions(self):
@@ -1595,7 +1617,10 @@ def read(self, partition):  # noqa: ARG002
             yield batch
 
     spark.dataSource.register(SQLInsertDataSource)
-    spark.sql(f"CREATE TABLE {table_name} USING {table_name}")
+    spark.sql(
+        f"CREATE TABLE {table_name} USING {table_name} "
+        f"OPTIONS (state_path '{escape_sql_string_literal(str(state_path))}')"
+    )
 
     try:
         # Read should work
@@ -1605,7 +1630,8 @@ def read(self, partition):  # noqa: ARG002
 
         # INSERT INTO should trigger the writer
         spark.sql(f"INSERT INTO {table_name} VALUES (2, 'new'), (3, 'another')")  # noqa: S608
-        assert SQLInsertWriter._commit_messages is not None  # noqa: SLF001
-        assert len(SQLInsertWriter._commit_messages) > 0  # noqa: SLF001
+        commit_messages = _read_test_state(state_path).get("commit_messages")
+        assert commit_messages is not None
+        assert len(commit_messages) > 0
     finally:
         spark.sql(f"DROP TABLE IF EXISTS {table_name}")
diff --git a/python/pysail/tests/spark/datasource/test_text.py b/python/pysail/tests/spark/datasource/test_text.py
index 052795ce2..2111e12e8 100644
--- a/python/pysail/tests/spark/datasource/test_text.py
+++ b/python/pysail/tests/spark/datasource/test_text.py
@@ -23,6 +23,17 @@ def test_text_read_write_basic(spark, sample_df, tmp_path):
     assert sorted(joined_df.collect(), key=safe_sort_key) == sorted(read_df.collect(), key=safe_sort_key)
 
 
+def test_text_path_glob_filter(spark, tmp_path):
+    path = tmp_path / "text_path_glob_filter"
+    path.mkdir()
+    (path / "keep.txt").write_text("keep\n")
+    (path / "drop.txt").write_text("drop\n")
+
+    df = spark.read.option("pathGlobFilter", "keep.*").text(str(path))
+
+    assert df.collect() == [Row(value="keep")]
+
+
 def test_text_read_write_compressed(spark, sample_df, tmp_path):
     path = str(tmp_path / "text_compressed_gzip")
     sample_df = sample_df.select("col1")
@@ -93,3 +104,25 @@ def test_text_read_projections(spark, sample_df, tmp_path):
     values = [r.value for r in projected_df.collect()]
     expected = [r["col1"] if r["col1"] is not None else "" for r in sample_df.collect()]
     assert sorted(values) == sorted(expected)
+
+
+def test_text_read_partition_columns(spark, tmp_path):
+    root = tmp_path / "text_partitioned"
+    (root / "country=us" / "city=sf").mkdir(parents=True)
+    (root / "country=us" / "city=ny").mkdir(parents=True)
+    (root / "country=ca" / "city=to").mkdir(parents=True)
+    (root / "country=us" / "city=sf" / "part-0.txt").write_text("golden\nbridge\n", encoding="utf-8")
+    (root / "country=us" / "city=ny" / "part-0.txt").write_text("hudson\n", encoding="utf-8")
+    (root / "country=ca" / "city=to" / "part-0.txt").write_text("harbour\n", encoding="utf-8")
+
+    read_df = spark.read.text(str(root)).select("value", "country", "city")
+
+    rows = {(row.value, row.country, row.city) for row in read_df.collect()}
+    assert rows == {
+        ("golden", "us", "sf"),
+        ("bridge", "us", "sf"),
+        ("hudson", "us", "ny"),
+        ("harbour", "ca", "to"),
+    }
+    expected_us_rows = 3
+    assert read_df.filter("country = 'us'").count() == expected_us_rows
diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/column_mapping.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/column_mapping.yaml
index dc451c65d..6ba6a3a41 100644
--- a/python/pysail/tests/spark/delta/__snapshots__/features/column_mapping.yaml
+++ b/python/pysail/tests/spark/delta/__snapshots__/features/column_mapping.yaml
@@ -189,3 +189,108 @@ data:
     - columnMapping
     writerFeatures:
     - columnMapping
+---
+name: "test_mergeschema_allocates_a_new_id_after_array_of_array_of_struct"
+data:
+  metaData:
+    configuration:
+      delta.columnMapping.maxColumnId: '3'
+      delta.columnMapping.mode: name
+    createdTime: 
+    description: null
+    format:
+      options: {}
+      provider: parquet
+    id: 
+    name: null
+    partitionColumns: []
+    schemaString:
+      fields:
+      - metadata:
+          delta.columnMapping.id: 1
+          delta.columnMapping.physicalName: 
+        name: matrix
+        nullable: true
+        type:
+          containsNull: true
+          elementType:
+            containsNull: true
+            elementType:
+              fields:
+              - metadata:
+                  delta.columnMapping.id: 2
+                  delta.columnMapping.physicalName: 
+                name: value
+                nullable: true
+                type: long
+              type: struct
+            type: array
+          type: array
+      - metadata:
+          delta.columnMapping.id: 3
+          delta.columnMapping.physicalName: 
+        name: label
+        nullable: false
+        type: string
+      type: struct
+  protocol:
+    minReaderVersion: 3
+    minWriterVersion: 7
+    readerFeatures:
+    - columnMapping
+    writerFeatures:
+    - columnMapping
+---
+name: "test_writing_a_scalar_idmapped_table_strips_physical_read_metadata"
+data:
+  metaData:
+    configuration: {}
+    createdTime: 
+    description: null
+    format:
+      options: {}
+      provider: parquet
+    id: 
+    name: null
+    partitionColumns: []
+    schemaString:
+      fields:
+      - metadata: {}
+        name: id
+        nullable: true
+        type: long
+      - metadata: {}
+        name: label
+        nullable: true
+        type: string
+      type: struct
+  protocol:
+    minReaderVersion: 1
+    minWriterVersion: 2
+---
+name: "test_writing_a_scalar_namemapped_table_strips_physical_read_metadata"
+data:
+  metaData:
+    configuration: {}
+    createdTime: 
+    description: null
+    format:
+      options: {}
+      provider: parquet
+    id: 
+    name: null
+    partitionColumns: []
+    schemaString:
+      fields:
+      - metadata: {}
+        name: id
+        nullable: true
+        type: long
+      - metadata: {}
+        name: label
+        nullable: true
+        type: string
+      type: struct
+  protocol:
+    minReaderVersion: 1
+    minWriterVersion: 2
diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml
index 2d4ac11e0..9d8710cf6 100644
--- a/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml
+++ b/python/pysail/tests/spark/delta/__snapshots__/features/delete.yaml
@@ -434,6 +434,91 @@ data:
                           ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
                             RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
                               DataSourceExec: file_groups={2 groups: [[/delete_explain_codegen/_delta_log/00000000000000000000.json], [/delete_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=2
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    DataSourceExec: file_groups={2 groups: [[/delete_explain_codegen/_delta_log/00000000000000000000.json], [/delete_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[__sail_delta_replay_path@7], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 2 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=2
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    DataSourceExec: file_groups={2 groups: [[/delete_explain_codegen/_delta_log/00000000000000000000.json], [/delete_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=2, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[__sail_delta_replay_path@7], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 4 ===
+    inputs=[StageInput(stage=3, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    DeltaDiscoveryExec(table_path=file:///delete_explain_codegen/)
+      FilterExec: (get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Engineering AND get_field(stats_parsed@7, maxValues, department) >= Engineering OR get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Marketing AND get_field(stats_parsed@7, maxValues, department) >= Marketing) AND (get_field(stats_parsed@7, nullCount, active) IS NULL OR get_field(stats_parsed@7, numRecords) IS NULL OR get_field(stats_parsed@7, nullCount, active) < get_field(stats_parsed@7, numRecords))
+        DeltaMetadataStatsExec(output=stats_parsed, columns=[id, name, age, department, salary])
+          DeltaLogReplayExec(table_path=file:///delete_explain_codegen/, version=1)
+            SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
+              StageInputExec: input=0, partitioning=Hash([__sail_delta_replay_path@7], 4)
+  
+    === stage 5 ===
+    inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=4, mode=Shuffle)]
+    mode=Pipelined
+    partitions=5
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    UnionExec
+      DeltaWriterExec(table_path=file:///delete_explain_codegen/)
+        FilterExec: NOT salary@4 + 1 > 70000 AND (department@3 = Engineering OR department@3 = Marketing) AND active@5 IS NOT NULL
+          DeltaScanByAddsExec(table_path=file:///delete_explain_codegen/, version=1, projection=None, limit=None, pushdown=false)
+            DeltaDiscoveryExec(table_path=file:///delete_explain_codegen/)
+              FilterExec: (get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Engineering AND get_field(stats_parsed@7, maxValues, department) >= Engineering OR get_field(stats_parsed@7, minValues, department) IS NULL OR get_field(stats_parsed@7, maxValues, department) IS NULL OR get_field(stats_parsed@7, minValues, department) <= Marketing AND get_field(stats_parsed@7, maxValues, department) >= Marketing) AND (get_field(stats_parsed@7, nullCount, active) IS NULL OR get_field(stats_parsed@7, numRecords) IS NULL OR get_field(stats_parsed@7, nullCount, active) < get_field(stats_parsed@7, numRecords))
+                DeltaMetadataStatsExec(output=stats_parsed, columns=[id, name, age, department, salary])
+                  DeltaLogReplayExec(table_path=file:///delete_explain_codegen/, version=1)
+                    SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
+                      StageInputExec: input=0, partitioning=Hash([__sail_delta_replay_path@7], 4)
+      DeltaRemoveActionsExec
+        StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+  
+    === stage 6 ===
+    inputs=[StageInput(stage=5, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Driver
+    DeltaCommitExec(table_path=file:///delete_explain_codegen/)
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 7 ===
+    inputs=[StageInput(stage=6, mode=Forward)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=UnknownPartitioning(1)
 ---
 name: "test_explain_includes_json_delta_log_commits_when_present"
 data:
diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/log_scan_parallel.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/log_scan_parallel.yaml
index 77acb7bed..7b0de69ba 100644
--- a/python/pysail/tests/spark/delta/__snapshots__/features/log_scan_parallel.yaml
+++ b/python/pysail/tests/spark/delta/__snapshots__/features/log_scan_parallel.yaml
@@ -11,7 +11,7 @@ data:
             UnionExec
               UnionExec
                 ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@6 ELSE id@3 END as id, CASE WHEN __common_expr_2@1 THEN __sail_src_value@7 WHEN __common_expr_1@0 THEN __sail_src_value@7 ELSE value@4 END as value, CASE WHEN __common_expr_3@2 THEN 3 WHEN __common_expr_2@1 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                  ProjectionExec: expr=[__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_1, __sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
+                  ProjectionExec: expr=[(__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
                     ProjectionExec: expr=[__sail_merge_target_row_present@3 IS NULL as __common_expr_4, id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_merge_target_row_present@3 as __sail_merge_target_row_present, __sail_src_id@4 as __sail_src_id, __sail_src_value@5 as __sail_src_value, __sail_merge_source_row_present@6 as __sail_merge_source_row_present]
                       FilterExec: __sail_file_path@2 IS NULL AND (__sail_merge_target_row_present@3 IS NOT NULL OR __sail_merge_target_row_present@3 IS NULL AND __sail_merge_source_row_present@6 IS NOT NULL)
                         ProjectionExec: expr=[id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_merge_target_row_present@4 as __sail_merge_target_row_present, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value, __sail_merge_source_row_present@7 as __sail_merge_source_row_present]
@@ -43,7 +43,7 @@ data:
                                       DataSourceExec: file_groups={3 groups: [[/logscan_parallel/part-.snappy.parquet], [/logscan_parallel/part-.snappy.parquet], [/logscan_parallel/part-.snappy.parquet]]}, projection=[id, value, __sail_file_path], file_type=parquet
                 UnionExec
                   ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@6 ELSE id@3 END as id, CASE WHEN __common_expr_2@1 THEN __sail_src_value@7 WHEN __common_expr_1@0 THEN __sail_src_value@7 ELSE value@4 END as value, __sail_file_path@5 as __sail_file_path, CASE WHEN __common_expr_3@2 THEN 3 WHEN __common_expr_2@1 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_1, __sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
+                    ProjectionExec: expr=[(__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
                       ProjectionExec: expr=[__sail_merge_target_row_present@3 IS NULL as __common_expr_4, id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_merge_target_row_present@3 as __sail_merge_target_row_present, __sail_src_id@4 as __sail_src_id, __sail_src_value@5 as __sail_src_value, __sail_merge_source_row_present@6 as __sail_merge_source_row_present]
                         FilterExec: __sail_merge_target_row_present@3 IS NOT NULL OR __sail_merge_target_row_present@3 IS NULL AND __sail_merge_source_row_present@6 IS NOT NULL
                           ProjectionExec: expr=[id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_merge_target_row_present@4 as __sail_merge_target_row_present, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value, __sail_merge_source_row_present@7 as __sail_merge_source_row_present]
diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml
index 66505cfc9..c6a5c641b 100644
--- a/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml
+++ b/python/pysail/tests/spark/delta/__snapshots__/features/merge.yaml
@@ -59,7 +59,7 @@ data:
     initial_logical_plan:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_id ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_category ELSE category END AS category, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN __sail_src_amount + Int32(1) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN Int32(6) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("stale") THEN Int32(5) WHEN Boolean(false) THEN Int32(7) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN Int32(6) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("stale") IS TRUE AND NOT __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN Int32(5) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
           Filter: __sail_merge_target_row_present IS NOT NULL OR NOT __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL
             MergeCardinalityCheck: target_row_id_col=__sail_merge_target_row_id, target_present_col=__sail_merge_target_row_present, source_present_col=__sail_merge_source_row_present
               Full Join: id = __sail_src_id Filter: category IS NOT NULL
@@ -148,7 +148,7 @@ data:
     logical_plan after simplify_expressions:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_id ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_category ELSE category END AS category, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN __sail_src_amount + Int32(1) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN Int32(6) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN Int32(6) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("stale") IS TRUE AND NOT __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN Int32(5) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
           Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
             MergeCardinalityCheck: target_row_id_col=__sail_merge_target_row_id, target_present_col=__sail_merge_target_row_present, source_present_col=__sail_merge_source_row_present
               Full Join: id = __sail_src_id Filter: category IS NOT NULL
@@ -255,7 +255,7 @@ data:
     logical_plan after push_down_filter:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_id ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_category ELSE category END AS category, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN __sail_src_amount + Int32(1) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") THEN Int32(6) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN Int32(6) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("stale") IS TRUE AND NOT __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND category = Utf8("vip") IS TRUE THEN Int32(5) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
           Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
             MergeCardinalityCheck: target_row_id_col=__sail_merge_target_row_id, target_present_col=__sail_merge_target_row_present, source_present_col=__sail_merge_source_row_present
               Full Join: id = __sail_src_id Filter: category IS NOT NULL
@@ -326,8 +326,8 @@ data:
     logical_plan after common_sub_expression_eliminate:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_merge_target_row_id, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_merge_target_row_id, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_merge_target_row_id, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_id, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
@@ -397,8 +397,8 @@ data:
     logical_plan after optimize_projections:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
@@ -517,8 +517,8 @@ data:
     logical_plan after push_down_filter:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Projection: id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
@@ -589,8 +589,8 @@ data:
     logical_plan after optimize_projections:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
@@ -707,8 +707,8 @@ data:
     logical_plan after push_down_filter:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Projection: id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
@@ -777,8 +777,8 @@ data:
     logical_plan after optimize_projections:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
@@ -838,8 +838,8 @@ data:
     logical_plan:
     RowLevelWrite: command=Merge, target=delta_merge_explain_codegen, format=DELTA, matched=2, not_matched=1, not_matched_by_source=1
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_1 THEN __sail_src_category ELSE category END AS category, CASE WHEN __common_expr_2 THEN __sail_src_amount + Int32(1) WHEN __common_expr_1 THEN __sail_src_amount ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN spark_concat(__sail_src_note, Utf8("_"), id) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN spark_concat(note, Utf8("_orphan")) WHEN __common_expr_1 THEN spark_concat(Utf8("insert_"), __sail_src_note) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") THEN Int32(8) WHEN __common_expr_5 AND category = Utf8("stale") THEN Int32(5) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_1 THEN CAST(__sail_src_category AS Utf8) ELSE category END AS category, CASE WHEN __common_expr_2 THEN CAST(__sail_src_amount + Int32(1) AS Int32) WHEN __common_expr_1 THEN CAST(__sail_src_amount AS Int32) ELSE amount END AS amount, CASE WHEN __common_expr_2 THEN CAST(spark_concat(__sail_src_note, Utf8("_"), id) AS Utf8) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN CAST(spark_concat(note, Utf8("_orphan")) AS Utf8) WHEN __common_expr_1 THEN CAST(spark_concat(Utf8("insert_"), __sail_src_note) AS Utf8) ELSE note END AS note, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_4 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) WHEN __common_expr_5 AND category = Utf8("stale") IS TRUE AND NOT __common_expr_2 THEN Int32(5) WHEN __common_expr_3 AND __sail_merge_source_row_present IS NULL AND category = Utf8("stale") IS TRUE THEN Int32(8) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_6 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __common_expr_7 AND category = Utf8("vip") IS TRUE AS __common_expr_2, __common_expr_8 AS __common_expr_3, __common_expr_6 AS __common_expr_4, __common_expr_7 AS __common_expr_5, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_6, __common_expr_9 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_7, __common_expr_9 AS __common_expr_8, id, category, amount, note, __sail_file_path, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
               Projection: __sail_merge_target_row_present IS NOT NULL AS __common_expr_9, id, category, amount, note, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_category, __sail_src_amount, __sail_src_note, __sail_merge_source_row_present
                 Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
@@ -902,8 +902,8 @@ data:
               UnionExec
                 FilterExec: __sail_file_path@4 IS NULL
                   UnionExec
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                             FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -914,8 +914,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -943,8 +942,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -952,8 +950,8 @@ data:
                                             FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
                                               DataSourceExec: partitions=1, partition_sizes=[1]
                     UnionExec
-                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                        ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                        ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                               FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -964,8 +962,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                       ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                         ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                           ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -995,8 +992,7 @@ data:
                                     MonotonicIdExec: col=__sail_merge_target_row_id
                                       ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                         ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                          ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                   ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                     ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                       ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -1020,8 +1016,8 @@ data:
               UnionExec, statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:)]]
                 FilterExec: __sail_file_path@4 IS NULL, statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:)]]
                   UnionExec, statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:)]]
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]: Min=Exact(Int64(NULL)) Max=Exact(Int64(NULL)) Sum=Exact(Int64(NULL)) Distinct=Exact(1) ScanBytes=Exact())]]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[6]: Null=Inexact(0)),(Col[7]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]:),(Col[10]: Null=Inexact(0)),(Col[11]: Null=Inexact(0)),(Col[12]: Null=Inexact(0)),(Col[13]: Null=Inexact(0)),(Col[14]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]: Min=Exact(Int64(NULL)) Max=Exact(Int64(NULL)) Sum=Exact(Int64(NULL)) Distinct=Exact(1) ScanBytes=Exact())]]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[6]: Null=Inexact(0)),(Col[7]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]:),(Col[10]: Null=Inexact(0)),(Col[11]: Null=Inexact(0)),(Col[12]: Null=Inexact(0)),(Col[13]: Null=Inexact(0)),(Col[14]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[4]: Null=Inexact(0)),(Col[5]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[6]: Null=Inexact(0)),(Col[7]:),(Col[8]: Null=Inexact(0)),(Col[9]: Null=Inexact(0)),(Col[10]: Null=Inexact(0)),(Col[11]: Null=Inexact(0)),(Col[12]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[4]: Null=Inexact(0)),(Col[5]:),(Col[6]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1)),(Col[7]: Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]: Null=Inexact(0)),(Col[10]: Null=Inexact(0)),(Col[11]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
                             FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL, statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[3]: Null=Inexact(0)),(Col[4]:),(Col[5]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1)),(Col[6]: Null=Inexact(0)),(Col[7]: Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]: Null=Inexact(0)),(Col[10]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
@@ -1032,8 +1028,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:),(Col[5]:)]]
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0)),(Col[4]: Min=Exact(Boolean(true)) Max=Exact(Boolean(true)) Null=Exact(0) Distinct=Exact(1))]]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
@@ -1061,8 +1056,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:),(Col[5]:)]]
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0)),(Col[4]: Min=Exact(Boolean(true)) Max=Exact(Boolean(true)) Null=Exact(0) Distinct=Exact(1))]]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
@@ -1070,8 +1064,8 @@ data:
                                             FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
                                               DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Null=Exact(0)),(Col[3]: Null=Exact(0))]]
                     UnionExec, statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:)]]
-                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]: Min=Exact(Int64(NULL)) Max=Exact(Int64(NULL)) Sum=Exact(Int64(NULL)) Distinct=Exact(1) ScanBytes=Exact())]]
-                        ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[6]: Null=Inexact(0)),(Col[7]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]:),(Col[10]: Null=Inexact(0)),(Col[11]: Null=Inexact(0)),(Col[12]: Null=Inexact(0)),(Col[13]: Null=Inexact(0)),(Col[14]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
+                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]: Min=Exact(Int64(NULL)) Max=Exact(Int64(NULL)) Sum=Exact(Int64(NULL)) Distinct=Exact(1) ScanBytes=Exact())]]
+                        ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[6]: Null=Inexact(0)),(Col[7]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]:),(Col[10]: Null=Inexact(0)),(Col[11]: Null=Inexact(0)),(Col[12]: Null=Inexact(0)),(Col[13]: Null=Inexact(0)),(Col[14]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
                           ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[4]: Null=Inexact(0)),(Col[5]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[6]: Null=Inexact(0)),(Col[7]:),(Col[8]: Null=Inexact(0)),(Col[9]: Null=Inexact(0)),(Col[10]: Null=Inexact(0)),(Col[11]: Null=Inexact(0)),(Col[12]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
                             ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]:),(Col[1]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[4]: Null=Inexact(0)),(Col[5]:),(Col[6]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1)),(Col[7]: Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]: Null=Inexact(0)),(Col[10]: Null=Inexact(0)),(Col[11]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
                               FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL, statistics=[Rows=Inexact(1), Bytes=Absent, [(Col[0]: Min=Inexact(Int32(1)) Max=Inexact(Int32(3)) Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Min=Inexact(Int32(10)) Max=Inexact(Int32(30)) Null=Inexact(0)),(Col[3]: Null=Inexact(0)),(Col[4]:),(Col[5]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1)),(Col[6]: Null=Inexact(0)),(Col[7]: Null=Inexact(0)),(Col[8]: Null=Inexact(0)),(Col[9]: Null=Inexact(0)),(Col[10]: Min=Inexact(Boolean(true)) Max=Inexact(Boolean(true)) Null=Inexact(0) Distinct=Inexact(1))]]
@@ -1082,8 +1076,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:),(Col[5]:)]]
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                       ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0)),(Col[4]: Min=Exact(Boolean(true)) Max=Exact(Boolean(true)) Null=Exact(0) Distinct=Exact(1))]]
                                         ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
                                           ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
@@ -1113,8 +1106,7 @@ data:
                                     MonotonicIdExec: col=__sail_merge_target_row_id, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:),(Col[5]:)]]
                                       ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                         ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                          ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
-                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
+                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Min=Exact(Int32(1)) Max=Exact(Int32(3)) Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]: Min=Exact(Int32(10)) Max=Exact(Int32(30)) Null=Exact(0)),(Col[3]: Null=Exact(0)),(Col[4]:)]]
                                   ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0)),(Col[4]: Min=Exact(Boolean(true)) Max=Exact(Boolean(true)) Null=Exact(0) Distinct=Exact(1))]]
                                     ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
                                       ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0)),(Col[2]: Null=Inexact(0)),(Col[3]: Null=Inexact(0))]]
@@ -1138,8 +1130,8 @@ data:
               UnionExec, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
                 FilterExec: __sail_file_path@4 IS NULL, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
                   UnionExec, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], schema=[__common_expr_1:Boolean, __common_expr_2:Boolean;N, __common_expr_3:Boolean, __common_expr_4:Boolean, __common_expr_5:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], schema=[__common_expr_1:Boolean, __common_expr_2:Boolean;N, __common_expr_3:Boolean, __common_expr_4:Boolean, __common_expr_5:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present], schema=[__common_expr_6:Boolean, __common_expr_7:Boolean, __common_expr_8:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present], schema=[__common_expr_9:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_present:Boolean;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
                             FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_present:Boolean;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
@@ -1150,8 +1142,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_id:Int64]
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], schema=[#0:Int32;N, #1:Utf8;N, #2:Int32;N, #3:Utf8;N, __sail_file_path:Utf8;N]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], schema=[__sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
@@ -1179,8 +1170,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_id:Int64]
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], schema=[#0:Int32;N, #1:Utf8;N, #2:Int32;N, #3:Utf8;N, __sail_file_path:Utf8;N]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], schema=[__sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
@@ -1188,8 +1178,8 @@ data:
                                             FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), schema=[column1:Int32;N, column2:Utf8;N, column3:Int32;N, column4:Utf8;N]
                                               DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Utf8;N, column3:Int32;N, column4:Utf8;N]
                     UnionExec, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
-                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
-                        ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], schema=[__common_expr_1:Boolean, __common_expr_2:Boolean;N, __common_expr_3:Boolean, __common_expr_4:Boolean, __common_expr_5:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
+                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_operation_type:Int32, __sail_merge_source_metric:Int64;N]
+                        ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present], schema=[__common_expr_1:Boolean, __common_expr_2:Boolean;N, __common_expr_3:Boolean, __common_expr_4:Boolean, __common_expr_5:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
                           ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present], schema=[__common_expr_6:Boolean, __common_expr_7:Boolean, __common_expr_8:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
                             ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present], schema=[__common_expr_9:Boolean, id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_present:Boolean;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
                               FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_present:Boolean;N, __sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean;N]
@@ -1200,8 +1190,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_id:Int64]
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], schema=[#0:Int32;N, #1:Utf8;N, #2:Int32;N, #3:Utf8;N, __sail_file_path:Utf8;N]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                       ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], schema=[__sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean]
                                         ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
                                           ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
@@ -1231,8 +1220,7 @@ data:
                                     MonotonicIdExec: col=__sail_merge_target_row_id, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N, __sail_merge_target_row_id:Int64]
                                       ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                         ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path], schema=[#0:Int32;N, #1:Utf8;N, #2:Int32;N, #3:Utf8;N, __sail_file_path:Utf8;N]
-                                          ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path], schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
-                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
+                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet, schema=[id:Int32;N, category:Utf8;N, amount:Int32;N, note:Utf8;N, __sail_file_path:Utf8;N]
                                   ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present], schema=[__sail_src_id:Int32;N, __sail_src_category:Utf8;N, __sail_src_amount:Int32;N, __sail_src_note:Utf8;N, __sail_merge_source_row_present:Boolean]
                                     ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
                                       ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11], schema=[#8:Int32;N, #9:Utf8;N, #10:Int32;N, #11:Utf8;N]
@@ -1257,8 +1245,8 @@ data:
                 UnionExec
                   FilterExec: __sail_file_path@4 IS NULL
                     UnionExec
-                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                        ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                        ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                               FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1269,8 +1257,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                       ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                         ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                           ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -1298,8 +1285,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                       ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                         ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                           ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -1307,8 +1293,8 @@ data:
                                               FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
                                                 DataSourceExec: partitions=1, partition_sizes=[1]
                       UnionExec
-                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                          ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                          ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                               ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                                 FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1319,8 +1305,7 @@ data:
                                           MonotonicIdExec: col=__sail_merge_target_row_id
                                             ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                               ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                         ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                           ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                             ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -1350,8 +1335,7 @@ data:
                                       MonotonicIdExec: col=__sail_merge_target_row_id
                                         ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                           ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                                     ProjectionExec: expr=[#8@0 as __sail_src_id, #9@1 as __sail_src_category, #10@2 as __sail_src_amount, #11@3 as __sail_src_note, true as __sail_merge_source_row_present]
                                       ProjectionExec: expr=[#8@0 as #8, #9@1 as #9, #10@2 as #10, #11@3 as #11]
                                         ProjectionExec: expr=[#4@0 as #8, #5@1 as #9, #6@2 as #10, #7@3 as #11]
@@ -1379,8 +1363,8 @@ data:
                 UnionExec
                   FilterExec: __sail_file_path@4 IS NULL
                     UnionExec
-                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                        ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                        ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                               FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1398,8 +1382,7 @@ data:
                                           MonotonicIdExec: col=__sail_merge_target_row_id
                                             ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                               ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                         AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                           AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
@@ -1428,11 +1411,10 @@ data:
                                           MonotonicIdExec: col=__sail_merge_target_row_id
                                             ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                               ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       UnionExec
-                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                          ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                          ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                               ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                                 FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1450,8 +1432,7 @@ data:
                                             MonotonicIdExec: col=__sail_merge_target_row_id
                                               ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                 ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                  ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                    DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                         ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                           AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                             AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
@@ -1482,8 +1463,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
                       SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, __sail_delta_log_version@9 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
                         RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
@@ -1504,8 +1484,8 @@ data:
               ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_operation_type@5 as __sail_operation_type, __sail_merge_source_metric@6 as __sail_merge_source_metric]
                 UnionExec
                   UnionExec
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                             FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -1523,8 +1503,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                       AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                         AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
@@ -1553,11 +1532,10 @@ data:
                                           MonotonicIdExec: col=__sail_merge_target_row_id
                                             ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                               ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       UnionExec
-                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                          ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                          ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                               ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                                 FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1575,8 +1553,7 @@ data:
                                             MonotonicIdExec: col=__sail_merge_target_row_id
                                               ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                 ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                  ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                    DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                         ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                           AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                             AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
@@ -1607,8 +1584,7 @@ data:
                                         MonotonicIdExec: col=__sail_merge_target_row_id
                                           ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                             ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                              ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                              DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
                       SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, __sail_delta_log_version@9 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
                         RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
@@ -1626,8 +1602,8 @@ data:
               ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_operation_type@5 as __sail_operation_type, __sail_merge_source_metric@6 as __sail_merge_source_metric]
                 UnionExec
                   UnionExec
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                             FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -1646,8 +1622,7 @@ data:
                                           MonotonicIdExec: col=__sail_merge_target_row_id
                                             ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                               ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                       AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                         CoalescePartitionsExec
@@ -1681,11 +1656,10 @@ data:
                                                 MonotonicIdExec: col=__sail_merge_target_row_id
                                                   ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                     ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       UnionExec
-                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                          ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                          ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                               ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                                 FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1704,8 +1678,7 @@ data:
                                               MonotonicIdExec: col=__sail_merge_target_row_id
                                                 ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                   ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                    DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                         ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                           AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                             CoalescePartitionsExec
@@ -1742,8 +1715,7 @@ data:
                                                 MonotonicIdExec: col=__sail_merge_target_row_id
                                                   ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                     ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
                         SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, __sail_delta_log_version@9 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
                           RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
@@ -1765,8 +1737,8 @@ data:
               ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_operation_type@5 as __sail_operation_type, __sail_merge_source_metric@6 as __sail_merge_source_metric]
                 UnionExec
                   UnionExec
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                             FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -1785,8 +1757,7 @@ data:
                                           MonotonicIdExec: col=__sail_merge_target_row_id
                                             ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                               ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                  DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                       AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                         CoalescePartitionsExec
@@ -1820,11 +1791,10 @@ data:
                                                 MonotonicIdExec: col=__sail_merge_target_row_id
                                                   ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                     ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       UnionExec
-                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                          ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                          ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                             ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                               ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                                 FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1843,8 +1813,7 @@ data:
                                               MonotonicIdExec: col=__sail_merge_target_row_id
                                                 ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                   ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                    DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                         ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                           AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                             CoalescePartitionsExec
@@ -1881,8 +1850,7 @@ data:
                                                 MonotonicIdExec: col=__sail_merge_target_row_id
                                                   ProjectionExec: expr=[#0@0 as id, #1@1 as category, #2@2 as amount, #3@3 as note, __sail_file_path@4 as __sail_file_path]
                                                     ProjectionExec: expr=[id@0 as #0, category@1 as #1, amount@2 as #2, note@3 as #3, __sail_file_path@4 as __sail_file_path]
-                                                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                       DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
                         SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
                           RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
@@ -1905,8 +1873,8 @@ data:
             DeltaWriterExec(table_path=file:///merge_explain_codegen/)
               UnionExec
                 UnionExec
-                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                    ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                           FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -1919,8 +1887,7 @@ data:
                                         DataSourceExec: partitions=1, partition_sizes=[1]
                                     ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                       MonotonicIdExec: col=__sail_merge_target_row_id
-                                        ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                   ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                     AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                       CoalescePartitionsExec
@@ -1943,11 +1910,10 @@ data:
                                         DataSourceExec: partitions=1, partition_sizes=[1]
                                     ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                       MonotonicIdExec: col=__sail_merge_target_row_id
-                                        ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                   UnionExec
-                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                      ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                    ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                      ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                           ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                             FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -1960,8 +1926,7 @@ data:
                                           DataSourceExec: partitions=1, partition_sizes=[1]
                                       ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                         MonotonicIdExec: col=__sail_merge_target_row_id
-                                          ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                       AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                         CoalescePartitionsExec
@@ -1987,8 +1952,7 @@ data:
                                           DataSourceExec: partitions=1, partition_sizes=[1]
                                       ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                         MonotonicIdExec: col=__sail_merge_target_row_id
-                                          ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                            DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                     DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
                       SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
                         RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
@@ -2004,8 +1968,8 @@ data:
           DeltaWriterExec(table_path=file:///merge_explain_codegen/)
             UnionExec
               UnionExec
-                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                  ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                  ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                     ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                         FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -2018,8 +1982,7 @@ data:
                                       DataSourceExec: partitions=1, partition_sizes=[1]
                                   ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                     MonotonicIdExec: col=__sail_merge_target_row_id
-                                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                 ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                   AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                     CoalescePartitionsExec
@@ -2042,11 +2005,10 @@ data:
                                       DataSourceExec: partitions=1, partition_sizes=[1]
                                   ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                     MonotonicIdExec: col=__sail_merge_target_row_id
-                                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                 UnionExec
-                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                    ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                           FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -2059,8 +2021,7 @@ data:
                                         DataSourceExec: partitions=1, partition_sizes=[1]
                                     ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                       MonotonicIdExec: col=__sail_merge_target_row_id
-                                        ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                   ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
                     AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
                       CoalescePartitionsExec
@@ -2086,8 +2047,7 @@ data:
                                         DataSourceExec: partitions=1, partition_sizes=[1]
                                     ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
                                       MonotonicIdExec: col=__sail_merge_target_row_id
-                                        ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path]
-                                          DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                   DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
                     SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
                       RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
@@ -2112,99 +2072,7 @@ data:
     SAME TEXT AS ABOVE
   
     physical_plan after ProjectionPushdown:
-    DeltaCommitExec(table_path=file:///merge_explain_codegen/)
-      CoalescePartitionsExec
-        UnionExec
-          DeltaWriterExec(table_path=file:///merge_explain_codegen/)
-            UnionExec
-              UnionExec
-                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                  ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
-                    ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
-                      ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
-                        FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
-                          ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
-                            MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
-                              RepartitionExec: partitioning=Hash([__sail_merge_target_row_id@5], 4), input_partitions=1
-                                HashJoinExec: mode=CollectLeft, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
-                                  ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
-                                    FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
-                                      DataSourceExec: partitions=1, partition_sizes=[1]
-                                  ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
-                                    MonotonicIdExec: col=__sail_merge_target_row_id
-                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
-                ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
-                  AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
-                    CoalescePartitionsExec
-                      AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
-                        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                          FilterExec: NULL IS NULL AND column3@1 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), projection=[]
-                            DataSourceExec: partitions=1, partition_sizes=[1]
-              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__sail_file_path@0, __sail_file_path@4)], projection=[id@1, category@2, amount@3, note@4, __sail_operation_type@6, __sail_merge_source_metric@7]
-                CoalescePartitionsExec
-                  AggregateExec: mode=FinalPartitioned, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
-                    RepartitionExec: partitioning=Hash([__sail_file_path@0], 4), input_partitions=4
-                      AggregateExec: mode=Partial, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
-                        FilterExec: __sail_merge_target_row_present@3 IS NOT NULL AND (__sail_merge_source_row_present@5 IS NOT NULL AND id@0 = __sail_src_id@4 AND category@1 IS NOT NULL AND (category@1 = vip OR category@1 = stale) OR __sail_merge_source_row_present@5 IS NULL AND category@1 = stale), projection=[__sail_file_path@2]
-                          ProjectionExec: expr=[id@0 as id, category@1 as category, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
-                            MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
-                              RepartitionExec: partitioning=Hash([__sail_merge_target_row_id@5], 4), input_partitions=1
-                                HashJoinExec: mode=CollectLeft, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
-                                  ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
-                                    FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
-                                      DataSourceExec: partitions=1, partition_sizes=[1]
-                                  ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
-                                    MonotonicIdExec: col=__sail_merge_target_row_id
-                                      DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
-                UnionExec
-                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
-                      ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
-                        ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
-                          FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
-                            ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
-                              MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
-                                RepartitionExec: partitioning=Hash([__sail_merge_target_row_id@5], 4), input_partitions=1
-                                  HashJoinExec: mode=CollectLeft, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
-                                    ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
-                                      FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
-                                        DataSourceExec: partitions=1, partition_sizes=[1]
-                                    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
-                                      MonotonicIdExec: col=__sail_merge_target_row_id
-                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
-                  ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
-                    AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
-                      CoalescePartitionsExec
-                        AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
-                          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-                            FilterExec: column3@1 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), projection=[]
-                              DataSourceExec: partitions=1, partition_sizes=[1]
-          DeltaRemoveActionsExec
-            CoalescePartitionsExec
-              DeltaDiscoveryExec(table_path=file:///merge_explain_codegen/)
-                HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__sail_file_path@0, __sail_file_path@0)], projection=[__sail_file_path@1, size_bytes@2, modification_time@3, _commit_version@4, _commit_timestamp@5, stats_json@6, deletionVector@7]
-                  CoalescePartitionsExec
-                    AggregateExec: mode=FinalPartitioned, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
-                      RepartitionExec: partitioning=Hash([__sail_file_path@0], 4), input_partitions=4
-                        AggregateExec: mode=Partial, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
-                          FilterExec: __sail_merge_target_row_present@3 IS NOT NULL AND (__sail_merge_source_row_present@5 IS NOT NULL AND id@0 = __sail_src_id@4 AND category@1 IS NOT NULL AND (category@1 = vip OR category@1 = stale) OR __sail_merge_source_row_present@5 IS NULL AND category@1 = stale), projection=[__sail_file_path@2]
-                            ProjectionExec: expr=[id@0 as id, category@1 as category, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
-                              MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
-                                RepartitionExec: partitioning=Hash([__sail_merge_target_row_id@5], 4), input_partitions=1
-                                  HashJoinExec: mode=CollectLeft, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
-                                    ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
-                                      FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
-                                        DataSourceExec: partitions=1, partition_sizes=[1]
-                                    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
-                                      MonotonicIdExec: col=__sail_merge_target_row_id
-                                        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
-                  DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
-                    SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
-                      RepartitionExec: partitioning=Hash([__sail_delta_replay_path@7], 4), input_partitions=4
-                        ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
-                          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
-                            DataSourceExec: file_groups={2 groups: [[/merge_explain_codegen/_delta_log/00000000000000000000.json], [/merge_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
-  
+    SAME TEXT AS ABOVE
   
     physical_plan after PushdownSort:
     SAME TEXT AS ABOVE
@@ -2234,8 +2102,8 @@ data:
           DeltaWriterExec(table_path=file:///merge_explain_codegen/)
             UnionExec
               UnionExec
-                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                  ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                  ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                     ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                         FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -2273,8 +2141,8 @@ data:
                                     MonotonicIdExec: col=__sail_merge_target_row_id
                                       DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                 UnionExec
-                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                    ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                           FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -2329,8 +2197,8 @@ data:
           DeltaWriterExec(table_path=file:///merge_explain_codegen/)
             UnionExec
               UnionExec
-                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                  ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                  ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                     ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                         FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
@@ -2368,8 +2236,8 @@ data:
                                     MonotonicIdExec: col=__sail_merge_target_row_id
                                       DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
                 UnionExec
-                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN __common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale THEN 8 WHEN __common_expr_5@4 AND category@6 = stale THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_1, __common_expr_7@1 AND category@4 = vip as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                    ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
                       ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
                         ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
                           FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
@@ -2415,6 +2283,291 @@ data:
                         ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
                           RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
                             DataSourceExec: file_groups={2 groups: [[/merge_explain_codegen/_delta_log/00000000000000000000.json], [/merge_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_src_id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
+      FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
+      MonotonicIdExec: col=__sail_merge_target_row_id
+        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=0, mode=Shuffle), StageInput(stage=1, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4)
+    placement=Worker
+    HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
+      StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1)
+      StageInputExec: input=1, partitioning=Hash([id@0], 1)
+  
+    === stage 3 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    FilterExec: NULL IS NULL AND column3@1 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), projection=[]
+      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 4 ===
+    inputs=[StageInput(stage=3, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 5 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_src_id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
+      FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 6 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
+      MonotonicIdExec: col=__sail_merge_target_row_id
+        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+  
+    === stage 7 ===
+    inputs=[StageInput(stage=5, mode=Shuffle), StageInput(stage=6, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4)
+    placement=Worker
+    HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
+      StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1)
+      StageInputExec: input=1, partitioning=Hash([id@0], 1)
+  
+    === stage 8 ===
+    inputs=[StageInput(stage=7, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[__sail_file_path@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
+      FilterExec: __sail_merge_target_row_present@3 IS NOT NULL AND (__sail_merge_source_row_present@5 IS NOT NULL AND id@0 = __sail_src_id@4 AND category@1 IS NOT NULL AND (category@1 = vip OR category@1 = stale) OR __sail_merge_source_row_present@5 IS NULL AND category@1 = stale), projection=[__sail_file_path@2]
+        ProjectionExec: expr=[id@0 as id, category@1 as category, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+          MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
+            StageInputExec: input=0, partitioning=Hash([__sail_merge_target_row_id@5], 4)
+  
+    === stage 9 ===
+    inputs=[StageInput(stage=8, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    AggregateExec: mode=FinalPartitioned, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
+      StageInputExec: input=0, partitioning=Hash([__sail_file_path@0], 4)
+  
+    === stage 10 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_src_id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
+      FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 11 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
+      MonotonicIdExec: col=__sail_merge_target_row_id
+        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+  
+    === stage 12 ===
+    inputs=[StageInput(stage=10, mode=Shuffle), StageInput(stage=11, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4)
+    placement=Worker
+    HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
+      StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1)
+      StageInputExec: input=1, partitioning=Hash([id@0], 1)
+  
+    === stage 13 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    FilterExec: column3@1 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4), projection=[]
+      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 14 ===
+    inputs=[StageInput(stage=13, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[], aggr=[count(1) as __sail_merge_source_metric]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 15 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_src_id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_category, column3@2 as __sail_src_amount, column4@3 as __sail_src_note, true as __sail_merge_source_row_present]
+      FilterExec: column3@2 + 1 > 10 AND (column1@0 = 1 OR column1@0 = 2 OR column1@0 = 4)
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 16 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[id@0], channels=1)
+    placement=Worker
+    ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_id@5 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
+      MonotonicIdExec: col=__sail_merge_target_row_id
+        DataSourceExec: file_groups={1 group: [[/merge_explain_codegen/part-.snappy.parquet]]}, projection=[id, category, amount, note, __sail_file_path], file_type=parquet
+  
+    === stage 17 ===
+    inputs=[StageInput(stage=15, mode=Shuffle), StageInput(stage=16, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[__sail_merge_target_row_id@5], channels=4)
+    placement=Worker
+    HashJoinExec: mode=Partitioned, join_type=Full, on=[(__sail_src_id@0, id@0)], filter=category@0 IS NOT NULL, projection=[id@5, category@6, amount@7, note@8, __sail_file_path@9, __sail_merge_target_row_id@10, __sail_merge_target_row_present@11, __sail_src_id@0, __sail_src_category@1, __sail_src_amount@2, __sail_src_note@3, __sail_merge_source_row_present@4]
+      StageInputExec: input=0, partitioning=Hash([__sail_src_id@0], 1)
+      StageInputExec: input=1, partitioning=Hash([id@0], 1)
+  
+    === stage 18 ===
+    inputs=[StageInput(stage=17, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[__sail_file_path@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
+      FilterExec: __sail_merge_target_row_present@3 IS NOT NULL AND (__sail_merge_source_row_present@5 IS NOT NULL AND id@0 = __sail_src_id@4 AND category@1 IS NOT NULL AND (category@1 = vip OR category@1 = stale) OR __sail_merge_source_row_present@5 IS NULL AND category@1 = stale), projection=[__sail_file_path@2]
+        ProjectionExec: expr=[id@0 as id, category@1 as category, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+          MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
+            StageInputExec: input=0, partitioning=Hash([__sail_merge_target_row_id@5], 4)
+  
+    === stage 19 ===
+    inputs=[StageInput(stage=18, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    AggregateExec: mode=FinalPartitioned, gby=[__sail_file_path@0 as __sail_file_path], aggr=[]
+      StageInputExec: input=0, partitioning=Hash([__sail_file_path@0], 4)
+  
+    === stage 20 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=2
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    DataSourceExec: file_groups={2 groups: [[/merge_explain_codegen/_delta_log/00000000000000000000.json], [/merge_explain_codegen/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
+  
+    === stage 21 ===
+    inputs=[StageInput(stage=20, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[__sail_delta_replay_path@7], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, stats) END as stats_json, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 22 ===
+    inputs=[StageInput(stage=19, mode=Broadcast), StageInput(stage=21, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    DeltaDiscoveryExec(table_path=file:///merge_explain_codegen/)
+      HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__sail_file_path@0, __sail_file_path@0)], projection=[__sail_file_path@1, size_bytes@2, modification_time@3, _commit_version@4, _commit_timestamp@5, stats_json@6, deletionVector@7]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+        DeltaLogReplayExec(table_path=file:///merge_explain_codegen/, version=1)
+          SortExec: expr=[__sail_delta_replay_path@7 ASC NULLS LAST, _commit_version@3 DESC NULLS LAST, __sail_delta_is_remove@8 ASC NULLS LAST], preserve_partitioning=[true]
+            StageInputExec: input=1, partitioning=Hash([__sail_delta_replay_path@7], 4)
+  
+    === stage 23 ===
+    inputs=[StageInput(stage=2, mode=Shuffle), StageInput(stage=4, mode=Shuffle), StageInput(stage=9, mode=Broadcast), StageInput(stage=12, mode=Shuffle), StageInput(stage=14, mode=Shuffle), StageInput(stage=22, mode=Shuffle)]
+    mode=Pipelined
+    partitions=11
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    UnionExec
+      DeltaWriterExec(table_path=file:///merge_explain_codegen/)
+        UnionExec
+          UnionExec
+            ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+              ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
+                    FilterExec: __sail_file_path@4 IS NULL AND (__sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL)
+                      ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+                        MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
+                          StageInputExec: input=0, partitioning=Hash([__sail_merge_target_row_id@5], 4)
+            ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
+              AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
+                StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+          HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__sail_file_path@0, __sail_file_path@4)], projection=[id@1, category@2, amount@3, note@4, __sail_operation_type@6, __sail_merge_source_metric@7]
+            StageInputExec: input=2, partitioning=UnknownPartitioning(1)
+            UnionExec
+              ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@10 ELSE id@5 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_category@11 ELSE category@6 END as category, CASE WHEN __common_expr_2@1 THEN __sail_src_amount@12 + 1 WHEN __common_expr_1@0 THEN __sail_src_amount@12 ELSE amount@7 END as amount, CASE WHEN __common_expr_2@1 THEN spark_concat(__sail_src_note@13, _, id@5) WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN spark_concat(note@8, _orphan) WHEN __common_expr_1@0 THEN spark_concat(insert_, __sail_src_note@13) ELSE note@8 END as note, __sail_file_path@9 as __sail_file_path, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND category@6 = stale) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@14 IS NULL AND category@6 = stale) IS NOT DISTINCT FROM true THEN 8 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@12 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND category@4 = vip) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, category@4 as category, amount@5 as amount, note@6 as note, __sail_file_path@7 as __sail_file_path, __sail_src_id@8 as __sail_src_id, __sail_src_category@9 as __sail_src_category, __sail_src_amount@10 as __sail_src_amount, __sail_src_note@11 as __sail_src_note, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[__sail_merge_target_row_present@6 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@11 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, category@2 as category, amount@3 as amount, note@4 as note, __sail_file_path@5 as __sail_file_path, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+                    ProjectionExec: expr=[__sail_merge_target_row_present@5 IS NOT NULL as __common_expr_9, id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_category@7 as __sail_src_category, __sail_src_amount@8 as __sail_src_amount, __sail_src_note@9 as __sail_src_note, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
+                      FilterExec: __sail_merge_target_row_present@5 IS NOT NULL OR __sail_merge_target_row_present@5 IS NULL AND __sail_merge_source_row_present@10 IS NOT NULL
+                        ProjectionExec: expr=[id@0 as id, category@1 as category, amount@2 as amount, note@3 as note, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_category@8 as __sail_src_category, __sail_src_amount@9 as __sail_src_amount, __sail_src_note@10 as __sail_src_note, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+                          MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
+                            StageInputExec: input=3, partitioning=Hash([__sail_merge_target_row_id@5], 4)
+              ProjectionExec: expr=[NULL as id, NULL as category, NULL as amount, NULL as note, NULL as __sail_file_path, 9 as __sail_operation_type, __sail_merge_source_metric@0 as __sail_merge_source_metric]
+                AggregateExec: mode=Final, gby=[], aggr=[count(1) as __sail_merge_source_metric]
+                  StageInputExec: input=4, partitioning=UnknownPartitioning(1)
+      DeltaRemoveActionsExec
+        StageInputExec: input=5, partitioning=UnknownPartitioning(1)
+  
+    === stage 24 ===
+    inputs=[StageInput(stage=23, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Driver
+    DeltaCommitExec(table_path=file:///merge_explain_codegen/)
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 25 ===
+    inputs=[StageInput(stage=24, mode=Forward)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=UnknownPartitioning(1)
 ---
 name: "test_explain_extended_does_not_include_mergecardinalitycheck_when_source_is_grouped_by_join_keys"
 data:
@@ -2422,7 +2575,7 @@ data:
     == Parsed Logical Plan ==
     RowLevelWrite: command=Merge, target=delta_merge_cardinality_skip, format=DELTA, matched=1, not_matched=1, not_matched_by_source=0
       Union
-        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_id ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_value WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_value ELSE value END AS value, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL THEN Int32(6) WHEN Boolean(false) THEN Int32(8) WHEN Boolean(false) THEN Int32(5) WHEN Boolean(false) THEN Int32(7) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_value AS Utf8) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_value AS Utf8) ELSE value END AS value, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN Int32(6) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
           Filter: __sail_merge_target_row_present IS NOT NULL OR NOT __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL
             Full Join: id = __sail_src_id
               Projection: id, value, __sail_file_path, Boolean(true) AS __sail_merge_target_row_present
@@ -2481,7 +2634,7 @@ data:
   
     RowLevelWrite: command=Merge, target=delta_merge_cardinality_skip, format=DELTA, matched=1, not_matched=1, not_matched_by_source=0
       Union
-        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_id ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_value WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL THEN __sail_src_value ELSE value END AS value, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL THEN Int32(6) WHEN Boolean(false) THEN Int32(8) WHEN Boolean(false) THEN Int32(5) WHEN Boolean(false) THEN Int32(7) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+        Projection: CASE WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_value AS Utf8) WHEN __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN CAST(__sail_src_value AS Utf8) ELSE value END AS value, __sail_file_path AS __sail_file_path, CASE WHEN __sail_merge_target_row_present IS NULL THEN Int32(3) WHEN __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AND Boolean(true) IS TRUE THEN Int32(6) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
           Filter: __sail_merge_target_row_present IS NOT NULL OR NOT __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL
             Full Join: id = __sail_src_id
               Projection: id, value, __sail_file_path, Boolean(true) AS __sail_merge_target_row_present
@@ -2539,8 +2692,8 @@ data:
     == Optimized Logical Plan ==
     RowLevelWrite: command=Merge, target=delta_merge_cardinality_skip, format=DELTA, matched=1, not_matched=1, not_matched_by_source=0
       Union
-        Projection: CASE WHEN __common_expr_1 THEN __sail_src_id ELSE id END AS id, CASE WHEN __common_expr_2 THEN __sail_src_value WHEN __common_expr_1 THEN __sail_src_value ELSE value END AS value, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_3 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
-          Projection: __common_expr_4 AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_1, __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL AS __common_expr_2, __common_expr_4 AS __common_expr_3, id, value, __sail_file_path, __sail_src_id, __sail_src_value
+        Projection: CASE WHEN __common_expr_1 THEN CAST(__sail_src_id AS Int32) ELSE id END AS id, CASE WHEN __common_expr_2 THEN CAST(__sail_src_value AS Utf8) WHEN __common_expr_1 THEN CAST(__sail_src_value AS Utf8) ELSE value END AS value, __sail_file_path AS __sail_file_path, CASE WHEN __common_expr_3 THEN Int32(3) WHEN __common_expr_2 THEN Int32(6) ELSE Int32(0) END AS __sail_operation_type, Int64(NULL) AS __sail_merge_source_metric
+          Projection: __common_expr_4 AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_1, __sail_merge_target_row_present IS NOT NULL AND __sail_merge_source_row_present IS NOT NULL IS TRUE AS __common_expr_2, __common_expr_4 AS __common_expr_3, id, value, __sail_file_path, __sail_src_id, __sail_src_value
             Projection: __sail_merge_target_row_present IS NULL AS __common_expr_4, id, value, __sail_file_path, __sail_merge_target_row_present, __sail_src_id, __sail_src_value, __sail_merge_source_row_present
               Filter: __sail_merge_target_row_present IS NOT NULL OR __sail_merge_target_row_present IS NULL AND __sail_merge_source_row_present IS NOT NULL
                 Full Join: id = __sail_src_id
@@ -2592,7 +2745,7 @@ data:
             UnionExec
               UnionExec
                 ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@6 ELSE id@3 END as id, CASE WHEN __common_expr_2@1 THEN __sail_src_value@7 WHEN __common_expr_1@0 THEN __sail_src_value@7 ELSE value@4 END as value, CASE WHEN __common_expr_3@2 THEN 3 WHEN __common_expr_2@1 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                  ProjectionExec: expr=[__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_1, __sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
+                  ProjectionExec: expr=[(__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
                     ProjectionExec: expr=[__sail_merge_target_row_present@3 IS NULL as __common_expr_4, id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_merge_target_row_present@3 as __sail_merge_target_row_present, __sail_src_id@4 as __sail_src_id, __sail_src_value@5 as __sail_src_value, __sail_merge_source_row_present@6 as __sail_merge_source_row_present]
                       FilterExec: __sail_file_path@2 IS NULL AND (__sail_merge_target_row_present@3 IS NOT NULL OR __sail_merge_target_row_present@3 IS NULL AND __sail_merge_source_row_present@6 IS NOT NULL)
                         HashJoinExec: mode=CollectLeft, join_type=Full, on=[(id@0, __sail_src_id@0)]
@@ -2631,7 +2784,7 @@ data:
                             DataSourceExec: file_groups={1 group: [[/merge_cardinality_skip/part-.snappy.parquet]]}, projection=[id, __sail_file_path], file_type=parquet, predicate=true AND DynamicFilter [ empty ]
                 UnionExec
                   ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@6 ELSE id@3 END as id, CASE WHEN __common_expr_2@1 THEN __sail_src_value@7 WHEN __common_expr_1@0 THEN __sail_src_value@7 ELSE value@4 END as value, __sail_file_path@5 as __sail_file_path, CASE WHEN __common_expr_3@2 THEN 3 WHEN __common_expr_2@1 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                    ProjectionExec: expr=[__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_1, __sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
+                    ProjectionExec: expr=[(__common_expr_4@0 AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__sail_merge_target_row_present@4 IS NOT NULL AND __sail_merge_source_row_present@7 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value]
                       ProjectionExec: expr=[__sail_merge_target_row_present@3 IS NULL as __common_expr_4, id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_merge_target_row_present@3 as __sail_merge_target_row_present, __sail_src_id@4 as __sail_src_id, __sail_src_value@5 as __sail_src_value, __sail_merge_source_row_present@6 as __sail_merge_source_row_present]
                         FilterExec: __sail_merge_target_row_present@3 IS NOT NULL OR __sail_merge_target_row_present@3 IS NULL AND __sail_merge_source_row_present@6 IS NOT NULL
                           HashJoinExec: mode=CollectLeft, join_type=Full, on=[(id@0, __sail_src_id@0)]
@@ -2683,7 +2836,7 @@ data:
     == Parsed Logical Plan ==
     RowLevelWrite: command=Merge, target=delta_merge_insert_only, format=DELTA, matched=0, not_matched=1, not_matched_by_source=0
       Union
-        Projection: CASE WHEN Boolean(true) THEN __sail_src_id ELSE NULL END AS id, CASE WHEN Boolean(true) THEN __sail_src_value ELSE NULL END AS value, CASE WHEN Boolean(true) THEN Int32(3) ELSE Int32(4) END AS __sail_operation_type
+        Projection: CASE WHEN Boolean(true) THEN CAST(__sail_src_id AS Int32) ELSE NULL END AS id, CASE WHEN Boolean(true) THEN CAST(__sail_src_value AS Utf8) ELSE NULL END AS value, CASE WHEN Boolean(true) THEN Int32(3) ELSE Int32(4) END AS __sail_operation_type
           LeftAnti Join: __sail_src_id = id
             Projection: s.#2 AS __sail_src_id, s.#3 AS __sail_src_value
               SubqueryAlias: s
@@ -2721,7 +2874,7 @@ data:
   
     RowLevelWrite: command=Merge, target=delta_merge_insert_only, format=DELTA, matched=0, not_matched=1, not_matched_by_source=0
       Union
-        Projection: CASE WHEN Boolean(true) THEN __sail_src_id ELSE CAST(NULL AS Int32) END AS id, CASE WHEN Boolean(true) THEN __sail_src_value ELSE CAST(NULL AS Utf8) END AS value, CASE WHEN Boolean(true) THEN Int32(3) ELSE Int32(4) END AS __sail_operation_type
+        Projection: CASE WHEN Boolean(true) THEN CAST(__sail_src_id AS Int32) ELSE CAST(NULL AS Int32) END AS id, CASE WHEN Boolean(true) THEN CAST(__sail_src_value AS Utf8) ELSE CAST(NULL AS Utf8) END AS value, CASE WHEN Boolean(true) THEN Int32(3) ELSE Int32(4) END AS __sail_operation_type
           LeftAnti Join: __sail_src_id = id
             Projection: s.#2 AS __sail_src_id, s.#3 AS __sail_src_value
               SubqueryAlias: s
@@ -2758,7 +2911,7 @@ data:
     == Optimized Logical Plan ==
     RowLevelWrite: command=Merge, target=delta_merge_insert_only, format=DELTA, matched=0, not_matched=1, not_matched_by_source=0
       Union
-        Projection: __sail_src_id AS id, __sail_src_value AS value, Int32(3) AS __sail_operation_type
+        Projection: CAST(__sail_src_id AS Int32) AS id, CAST(__sail_src_value AS Utf8) AS value, Int32(3) AS __sail_operation_type
           LeftAnti Join: __sail_src_id = id
             Projection: s.#2 AS __sail_src_id, s.#3 AS __sail_src_value
               SubqueryAlias: s
@@ -2848,8 +3001,8 @@ data:
         UnionExec
           DeltaWriterExec(table_path=file:///merge_dv/)
             UnionExec
-              ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@7 ELSE id@2 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_value@8 ELSE value@3 END as value, CASE WHEN __common_expr_1@0 THEN __sail_src_flag@9 ELSE flag@4 END as flag, CASE WHEN __common_expr_2@1 THEN 3 WHEN __sail_merge_target_row_present@6 IS NOT NULL AND __sail_merge_source_row_present@10 IS NOT NULL AND __sail_src_flag@9 = delete THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                ProjectionExec: expr=[__common_expr_3@0 AND __sail_merge_source_row_present@9 IS NOT NULL as __common_expr_1, __common_expr_3@0 as __common_expr_2, id@1 as id, value@2 as value, flag@3 as flag, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_value@7 as __sail_src_value, __sail_src_flag@8 as __sail_src_flag, __sail_merge_source_row_present@9 as __sail_merge_source_row_present]
+              ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@7 ELSE id@2 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_value@8 ELSE value@3 END as value, CASE WHEN __common_expr_1@0 THEN __sail_src_flag@9 ELSE flag@4 END as flag, CASE WHEN __common_expr_2@1 THEN 3 WHEN (__sail_merge_target_row_present@6 IS NOT NULL AND __sail_merge_source_row_present@10 IS NOT NULL AND __sail_src_flag@9 = delete) IS NOT DISTINCT FROM true THEN 5 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+                ProjectionExec: expr=[(__common_expr_3@0 AND __sail_merge_source_row_present@9 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, __common_expr_3@0 as __common_expr_2, id@1 as id, value@2 as value, flag@3 as flag, __sail_file_path@4 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_value@7 as __sail_src_value, __sail_src_flag@8 as __sail_src_flag, __sail_merge_source_row_present@9 as __sail_merge_source_row_present]
                   ProjectionExec: expr=[__sail_merge_target_row_present@4 IS NULL as __common_expr_3, id@0 as id, value@1 as value, flag@2 as flag, __sail_file_path@3 as __sail_file_path, __sail_merge_target_row_present@4 as __sail_merge_target_row_present, __sail_src_id@5 as __sail_src_id, __sail_src_value@6 as __sail_src_value, __sail_src_flag@7 as __sail_src_flag, __sail_merge_source_row_present@8 as __sail_merge_source_row_present]
                     FilterExec: __sail_file_path@3 IS NULL AND (__sail_merge_target_row_present@4 IS NOT NULL OR __sail_merge_target_row_present@4 IS NULL AND __sail_merge_source_row_present@8 IS NOT NULL)
                       ProjectionExec: expr=[id@0 as id, value@1 as value, flag@2 as flag, __sail_file_path@3 as __sail_file_path, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_id@7 as __sail_src_id, __sail_src_value@8 as __sail_src_value, __sail_src_flag@9 as __sail_src_flag, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
@@ -2938,7 +3091,7 @@ data:
                 UnionExec
                   UnionExec
                     ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@7 ELSE id@3 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_year@8 ELSE year@4 END as year, CASE WHEN __common_expr_2@1 THEN __sail_src_value@9 WHEN __common_expr_1@0 THEN __sail_src_value@9 ELSE value@5 END as value, CASE WHEN __common_expr_3@2 THEN 3 WHEN __common_expr_2@1 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                      ProjectionExec: expr=[__common_expr_4@0 AND __sail_merge_source_row_present@9 IS NOT NULL as __common_expr_1, __sail_merge_target_row_present@5 IS NOT NULL AND __sail_merge_source_row_present@9 IS NOT NULL as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, year@2 as year, value@3 as value, __sail_file_path@4 as __sail_file_path, __sail_src_id@6 as __sail_src_id, __sail_src_year@7 as __sail_src_year, __sail_src_value@8 as __sail_src_value]
+                      ProjectionExec: expr=[(__common_expr_4@0 AND __sail_merge_source_row_present@9 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__sail_merge_target_row_present@5 IS NOT NULL AND __sail_merge_source_row_present@9 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, year@2 as year, value@3 as value, __sail_file_path@4 as __sail_file_path, __sail_src_id@6 as __sail_src_id, __sail_src_year@7 as __sail_src_year, __sail_src_value@8 as __sail_src_value]
                         ProjectionExec: expr=[__sail_merge_target_row_present@4 IS NULL as __common_expr_4, id@0 as id, year@1 as year, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_merge_target_row_present@4 as __sail_merge_target_row_present, __sail_src_id@5 as __sail_src_id, __sail_src_year@6 as __sail_src_year, __sail_src_value@7 as __sail_src_value, __sail_merge_source_row_present@8 as __sail_merge_source_row_present]
                           FilterExec: __sail_file_path@3 IS NULL AND (__sail_merge_target_row_present@4 IS NOT NULL OR __sail_merge_target_row_present@4 IS NULL AND __sail_merge_source_row_present@8 IS NOT NULL)
                             ProjectionExec: expr=[id@0 as id, year@1 as year, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_year@7 as __sail_src_year, __sail_src_value@8 as __sail_src_value, __sail_merge_source_row_present@9 as __sail_merge_source_row_present]
@@ -2970,7 +3123,7 @@ data:
                                           DataSourceExec: file_groups={2 groups: [[/merge_explain_partition_pushdown/year=2023/part-.snappy.parquet], [/merge_explain_partition_pushdown/year=2024/part-.snappy.parquet]]}, projection=[id, year, value, __sail_file_path], file_type=parquet
                     UnionExec
                       ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@7 ELSE id@3 END as id, CASE WHEN __common_expr_1@0 THEN __sail_src_year@8 ELSE year@4 END as year, CASE WHEN __common_expr_2@1 THEN __sail_src_value@9 WHEN __common_expr_1@0 THEN __sail_src_value@9 ELSE value@5 END as value, __sail_file_path@6 as __sail_file_path, CASE WHEN __common_expr_3@2 THEN 3 WHEN __common_expr_2@1 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
-                        ProjectionExec: expr=[__common_expr_4@0 AND __sail_merge_source_row_present@9 IS NOT NULL as __common_expr_1, __sail_merge_target_row_present@5 IS NOT NULL AND __sail_merge_source_row_present@9 IS NOT NULL as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, year@2 as year, value@3 as value, __sail_file_path@4 as __sail_file_path, __sail_src_id@6 as __sail_src_id, __sail_src_year@7 as __sail_src_year, __sail_src_value@8 as __sail_src_value]
+                        ProjectionExec: expr=[(__common_expr_4@0 AND __sail_merge_source_row_present@9 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__sail_merge_target_row_present@5 IS NOT NULL AND __sail_merge_source_row_present@9 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_4@0 as __common_expr_3, id@1 as id, year@2 as year, value@3 as value, __sail_file_path@4 as __sail_file_path, __sail_src_id@6 as __sail_src_id, __sail_src_year@7 as __sail_src_year, __sail_src_value@8 as __sail_src_value]
                           ProjectionExec: expr=[__sail_merge_target_row_present@4 IS NULL as __common_expr_4, id@0 as id, year@1 as year, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_merge_target_row_present@4 as __sail_merge_target_row_present, __sail_src_id@5 as __sail_src_id, __sail_src_year@6 as __sail_src_year, __sail_src_value@7 as __sail_src_value, __sail_merge_source_row_present@8 as __sail_merge_source_row_present]
                             FilterExec: __sail_merge_target_row_present@4 IS NOT NULL OR __sail_merge_target_row_present@4 IS NULL AND __sail_merge_source_row_present@8 IS NOT NULL
                               ProjectionExec: expr=[id@0 as id, year@1 as year, value@2 as value, __sail_file_path@3 as __sail_file_path, __sail_merge_target_row_present@5 as __sail_merge_target_row_present, __sail_src_id@6 as __sail_src_id, __sail_src_year@7 as __sail_src_year, __sail_src_value@8 as __sail_src_value, __sail_merge_source_row_present@9 as __sail_merge_source_row_present]
diff --git a/python/pysail/tests/spark/delta/__snapshots__/features/read_path.yaml b/python/pysail/tests/spark/delta/__snapshots__/features/read_path.yaml
index 4569e820d..cce846d07 100644
--- a/python/pysail/tests/spark/delta/__snapshots__/features/read_path.yaml
+++ b/python/pysail/tests/spark/delta/__snapshots__/features/read_path.yaml
@@ -28,3 +28,10 @@ data:
                   ProjectionExec: expr=[CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END as __sail_file_path, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, size) END ELSE 0 END as size_bytes, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as modification_time, __sail_delta_log_version@2 as _commit_version, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, modificationTime) END ELSE 0 END as _commit_timestamp, CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, deletionVector) END as deletionVector, CASE WHEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END IS NOT NULL THEN CASE WHEN add@0 IS NOT NULL THEN get_field(add@0, path) END ELSE CASE WHEN remove@1 IS NOT NULL THEN get_field(remove@1, path) END END as __sail_delta_replay_path, remove@1 IS NOT NULL as __sail_delta_is_remove, __sail_delta_log_version@2 as __sail_delta_log_version]
                     RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
                       DataSourceExec: file_groups={2 groups: [[/delta_read_metadata/_delta_log/00000000000000000000.json], [/delta_read_metadata/_delta_log/00000000000000000001.json]]}, projection=[add, remove, __sail_delta_log_version], file_type=json
+---
+name: "test_explain_select_with_timestamp_uses_the_standard_driver_file_scan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as id, #3@1 as event_time]
+      DataSourceExec: file_groups={1 group: [[/delta_read_driver_timestamp/part-.snappy.parquet]]}, projection=[id@0 as #2, event_time@1 as #3], file_type=parquet
diff --git a/python/pysail/tests/spark/delta/conftest.py b/python/pysail/tests/spark/delta/conftest.py
index d21407737..a6bf02561 100644
--- a/python/pysail/tests/spark/delta/conftest.py
+++ b/python/pysail/tests/spark/delta/conftest.py
@@ -1,9 +1,46 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pyspark
 import pytest
+from pyspark.sql import SparkSession
 
 from pysail.testing.spark.utils.common import is_jvm_spark
+from pysail.testing.spark.utils.jvm import classic_spark_mode, delta_spark_maven_coordinate
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
 
 
 @pytest.fixture(scope="session", autouse=True)
 def skip_if_jvm_spark():
     if is_jvm_spark():
         pytest.skip("Delta Lake tests for JVM Spark")
+
+
+@pytest.fixture(scope="session")
+def delta_jvm_spark(tmp_path_factory: pytest.TempPathFactory) -> Generator[SparkSession, None, None]:
+    warehouse_dir = tmp_path_factory.mktemp("delta-jvm-warehouse")
+    try:
+        delta_package = delta_spark_maven_coordinate(pyspark.__version__)
+    except RuntimeError as error:
+        pytest.skip(str(error))
+    with classic_spark_mode():
+        spark = (
+            SparkSession.builder.master("local[1]")
+            .appName("delta-column-mapping-interop")
+            .config("spark.jars.packages", delta_package)
+            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+            .config(
+                "spark.sql.catalog.spark_catalog",
+                "org.apache.spark.sql.delta.catalog.DeltaCatalog",
+            )
+            .config("spark.sql.warehouse.dir", str(warehouse_dir))
+            .getOrCreate()
+        )
+        spark.conf.set("spark.sql.session.timeZone", "UTC")
+
+    yield spark
+
+    spark.stop()
diff --git a/python/pysail/tests/spark/delta/features/check_constraints.feature b/python/pysail/tests/spark/delta/features/check_constraints.feature
index d6a215149..b79dcac9a 100644
--- a/python/pysail/tests/spark/delta/features/check_constraints.feature
+++ b/python/pysail/tests/spark/delta/features/check_constraints.feature
@@ -207,3 +207,57 @@ Feature: Delta Lake CHECK Constraints
         WHEN NOT MATCHED THEN INSERT *
         """
       Then query error DELTA_VIOLATE_CONSTRAINT_WITH_VALUES
+
+  Rule: MERGE respects NOT NULL constraints
+
+    Background:
+      Given variable location for temporary directory delta_not_null_merge
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_not_null_merge_test
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_not_null_merge_test (
+          id INT NOT NULL,
+          value STRING
+        )
+        USING DELTA
+        LOCATION {{ location.sql }}
+        """
+
+    Scenario: MERGE inserts valid rows into a table with a NOT NULL column
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW src_delta_not_null_merge AS
+        SELECT 1 AS id, 'valid' AS value
+        """
+      Given statement
+        """
+        MERGE INTO delta_not_null_merge_test AS t
+        USING src_delta_not_null_merge AS s
+        ON t.id = s.id
+        WHEN NOT MATCHED THEN INSERT (id, value) VALUES (s.id, s.value)
+        """
+      When query
+        """
+        SELECT id, value FROM delta_not_null_merge_test
+        """
+      Then query result ordered
+        | id | value |
+        | 1  | valid |
+
+    Scenario: MERGE rejects null values for non-nullable target columns
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW src_delta_not_null_merge AS
+        SELECT CAST(NULL AS INT) AS id, 'invalid' AS value
+        """
+      When query
+        """
+        MERGE INTO delta_not_null_merge_test AS t
+        USING src_delta_not_null_merge AS s
+        ON t.id = s.id
+        WHEN NOT MATCHED THEN INSERT (id, value) VALUES (s.id, s.value)
+        """
+      Then query error DELTA_NOT_NULL_CONSTRAINT_VIOLATED.*column: id
diff --git a/python/pysail/tests/spark/delta/features/column_mapping.feature b/python/pysail/tests/spark/delta/features/column_mapping.feature
index 39e140fed..0f993f9d3 100644
--- a/python/pysail/tests/spark/delta/features/column_mapping.feature
+++ b/python/pysail/tests/spark/delta/features/column_mapping.feature
@@ -113,6 +113,99 @@ Feature: Delta Lake Column Mapping (DDL TBLPROPERTIES)
         """
       Then delta log first commit protocol and metadata matches snapshot
 
+  Rule: Column IDs remain unique through consecutive collection types
+    Background:
+      Given variable location for temporary directory cm_consecutive_arrays
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_cm_consecutive_arrays
+        """
+
+    Scenario: mergeSchema allocates a new ID after array of array of struct
+      Given statement template
+        """
+        CREATE TABLE delta_cm_consecutive_arrays (
+          matrix ARRAY>>
+        )
+        USING DELTA
+        LOCATION {{ location.sql }}
+        TBLPROPERTIES (
+          'delta.columnMapping.mode' = 'name'
+        )
+        """
+      Given append query to delta table in location with mergeSchema
+        """
+        SELECT
+          array(array(named_struct('value', CAST(2 AS BIGINT)))) AS matrix,
+          'new' AS label
+        """
+      Then delta log latest effective protocol and metadata matches snapshot
+
+  Rule: Scalar mapping metadata is not propagated to a non-mapped table
+    Background:
+      Given variable source_location for temporary directory cm_scalar_source
+      Given variable location for temporary directory cm_scalar_target
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_cm_scalar_source
+        """
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_cm_scalar_target
+        """
+
+    Scenario: Writing a scalar name-mapped table strips physical read metadata
+      Given statement template
+        """
+        CREATE TABLE delta_cm_scalar_source (
+          id BIGINT,
+          label STRING
+        )
+        USING DELTA
+        LOCATION {{ source_location.sql }}
+        TBLPROPERTIES (
+          'delta.columnMapping.mode' = 'name'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO delta_cm_scalar_source VALUES (1, 'a')
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_cm_scalar_target
+        USING DELTA
+        LOCATION {{ location.sql }}
+        AS SELECT * FROM delta_cm_scalar_source
+        """
+      Then delta log latest effective protocol and metadata matches snapshot
+
+    Scenario: Writing a scalar ID-mapped table strips physical read metadata
+      Given statement template
+        """
+        CREATE TABLE delta_cm_scalar_source (
+          id BIGINT,
+          label STRING
+        )
+        USING DELTA
+        LOCATION {{ source_location.sql }}
+        TBLPROPERTIES (
+          'delta.columnMapping.mode' = 'id'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO delta_cm_scalar_source VALUES (1, 'a')
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_cm_scalar_target
+        USING DELTA
+        LOCATION {{ location.sql }}
+        AS SELECT * FROM delta_cm_scalar_source
+        """
+      Then delta log latest effective protocol and metadata matches snapshot
+
   Rule: Column mapping preserves special characters in column names
     Background:
       Given variable location for temporary directory cm_special_chars
diff --git a/python/pysail/tests/spark/delta/features/merge.feature b/python/pysail/tests/spark/delta/features/merge.feature
index 6a005ed04..a9012d40e 100644
--- a/python/pysail/tests/spark/delta/features/merge.feature
+++ b/python/pysail/tests/spark/delta/features/merge.feature
@@ -1,5 +1,293 @@
 Feature: Delta Lake Merge
 
+  Rule: MERGE conditions must be deterministic
+
+    Scenario: A non-deterministic matched condition is rejected before writing
+      Given variable location for temporary directory delta_merge_nondeterministic
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_nondeterministic
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_nondeterministic (id INT, value STRING)
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_nondeterministic VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_nondeterministic_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      When query
+        """
+        MERGE INTO delta_merge_nondeterministic AS t
+        USING delta_merge_nondeterministic_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND rand() < 0.5 THEN UPDATE SET value = s.value
+        """
+      Then query error Non-deterministic expressions are not allowed in MERGE conditions
+      When query
+        """
+        SELECT id, value FROM delta_merge_nondeterministic
+        """
+      Then query result
+        | id | value |
+        | 1  | old   |
+
+  Rule: Internal MERGE columns cannot shadow table data
+
+    Scenario: A target column using an internal operation name is rejected clearly
+      Given variable location for temporary directory delta_merge_internal_column
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_internal_column
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_internal_column (
+          `__sail_operation_type` INT,
+          value STRING
+        )
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_internal_column VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_internal_column_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      When query
+        """
+        MERGE INTO delta_merge_internal_column AS t
+        USING delta_merge_internal_column_source AS s
+        ON t.`__sail_operation_type` = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      Then query error reserved internal MERGE column
+
+    Scenario: A legal target column does not collide with generated source aliases
+      Given variable location for temporary directory delta_merge_source_alias
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_source_alias
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_source_alias (`__sail_src_id` INT, value STRING)
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_source_alias VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_source_alias_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      Given statement
+        """
+        MERGE INTO delta_merge_source_alias AS t
+        USING delta_merge_source_alias_source AS s
+        ON t.`__sail_src_id` = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      When query
+        """
+        SELECT `__sail_src_id`, value FROM delta_merge_source_alias
+        """
+      Then query result
+        | __sail_src_id | value |
+        | 1             | new   |
+
+  Rule: MERGE assignments follow target schema semantics
+
+    Scenario: Star actions reject source columns missing from the target schema
+      Given variable location for temporary directory delta_merge_star_missing
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_star_missing
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_star_missing (id INT, value STRING, keep STRING)
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_star_missing VALUES (1, 'old', 'preserved')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_star_missing_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      When query
+        """
+        MERGE INTO delta_merge_star_missing AS t
+        USING delta_merge_star_missing_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET *
+        WHEN NOT MATCHED THEN INSERT *
+        """
+      Then query error Cannot resolve source column `keep` for MERGE \* action without schema evolution
+
+    Scenario: Assignments cast to target types and reject overflow
+      Given config spark.sql.ansi.enabled = true
+      Given variable location for temporary directory delta_merge_assignment_cast
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_assignment_cast
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_assignment_cast (id INT, value INT)
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_assignment_cast VALUES (1, 10)
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_assignment_cast_source AS
+        SELECT 1 AS id, CAST(20 AS BIGINT) AS value
+        """
+      Given statement
+        """
+        MERGE INTO delta_merge_assignment_cast AS t
+        USING delta_merge_assignment_cast_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      When query
+        """
+        SELECT id, value FROM delta_merge_assignment_cast
+        """
+      Then query result
+        | id | value |
+        | 1  | 20    |
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_assignment_cast_source AS
+        SELECT 1 AS id, CAST(2147483648 AS BIGINT) AS value
+        """
+      When query
+        """
+        MERGE INTO delta_merge_assignment_cast AS t
+        USING delta_merge_assignment_cast_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      Then query error (?i).*(cast|overflow).*
+      When query
+        """
+        SELECT id, value FROM delta_merge_assignment_cast
+        """
+      Then query result
+        | id | value |
+        | 1  | 20    |
+
+    Scenario: Duplicate target assignments are rejected
+      Given variable location for temporary directory delta_merge_duplicate_assignment
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_duplicate_assignment
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_duplicate_assignment (id INT, value STRING)
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_duplicate_assignment VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_duplicate_assignment_source AS
+        SELECT 1 AS id
+        """
+      When query
+        """
+        MERGE INTO delta_merge_duplicate_assignment AS t
+        USING delta_merge_duplicate_assignment_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = 'first', value = 'second'
+        """
+      Then query error Multiple assignments for MERGE target column
+
+    Scenario: Star matching honors case-sensitive resolution
+      Given config spark.sql.caseSensitive = true
+      Given variable location for temporary directory delta_merge_case_sensitive
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_case_sensitive
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_case_sensitive (`A` INT)
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_case_sensitive VALUES (1)
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_case_sensitive_source AS
+        SELECT 1 AS a
+        """
+      When query
+        """
+        MERGE INTO delta_merge_case_sensitive AS t
+        USING delta_merge_case_sensitive_source AS s
+        ON t.`A` = s.a
+        WHEN MATCHED THEN UPDATE SET *
+        """
+      Then query error Cannot resolve source column `A` for MERGE \* action without schema evolution
+
+    Scenario: Explicit inserts use target column defaults
+      Given variable location for temporary directory delta_merge_assignment_default
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_assignment_default
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_assignment_default (
+          id INT,
+          status STRING DEFAULT 'new'
+        )
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_assignment_default_source AS
+        SELECT 1 AS id
+        """
+      Given statement
+        """
+        MERGE INTO delta_merge_assignment_default AS t
+        USING delta_merge_assignment_default_source AS s
+        ON t.id = s.id
+        WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id)
+        """
+      When query
+        """
+        SELECT id, status FROM delta_merge_assignment_default
+        """
+      Then query result
+        | id | status |
+        | 1  | new    |
+
   Rule: Matched updates, deletes, and default inserts
     Background:
       Given variable location for temporary directory merge_basic
@@ -65,6 +353,69 @@ Feature: Delta Lake Merge
         | 2  | new   | update |
         | 4  | ins   | insert |
 
+  Rule: WHEN clauses use first-match semantics
+
+    Scenario: Overlapping matched and target-only clauses apply only their first action
+      Given variable location for temporary directory delta_merge_first_match
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_first_match
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_first_match (
+          id INT,
+          left_value STRING,
+          right_value STRING,
+          kind STRING
+        )
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_first_match VALUES
+          (1, 'old-left', 'old-right', 'delete-update'),
+          (2, 'old-left', 'old-right', 'partial-update'),
+          (3, 'old-left', 'old-right', 'source-update-delete'),
+          (4, 'old-left', 'old-right', 'source-delete-update')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW delta_merge_first_match_source AS
+        SELECT * FROM VALUES
+          (1, 'new-left', 'new-right', 'delete-update'),
+          (2, 'new-left', 'new-right', 'partial-update')
+        AS src(id, left_value, right_value, kind)
+        """
+      Given statement
+        """
+        MERGE INTO delta_merge_first_match AS t
+        USING delta_merge_first_match_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND s.kind = 'delete-update' THEN DELETE
+        WHEN MATCHED AND s.kind IN ('delete-update', 'partial-update') THEN
+          UPDATE SET left_value = s.left_value
+        WHEN MATCHED AND s.kind = 'partial-update' THEN
+          UPDATE SET right_value = s.right_value
+        WHEN MATCHED THEN DELETE
+        WHEN NOT MATCHED BY SOURCE AND t.kind = 'source-delete-update' THEN DELETE
+        WHEN NOT MATCHED BY SOURCE AND t.kind IN ('source-update-delete', 'source-delete-update') THEN
+          UPDATE SET left_value = 'source-left'
+        WHEN NOT MATCHED BY SOURCE AND t.kind = 'source-update-delete' THEN
+          UPDATE SET right_value = 'source-right'
+        WHEN NOT MATCHED BY SOURCE THEN DELETE
+        """
+      When query
+        """
+        SELECT id, left_value, right_value, kind
+        FROM delta_merge_first_match
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | left_value  | right_value | kind                 |
+        | 2  | new-left    | old-right   | partial-update       |
+        | 3  | source-left | old-right   | source-update-delete |
+
   Rule: Cardinality violation is rejected when multiple source rows match one target row
     Background:
       Given variable location for temporary directory merge_cardinality
@@ -749,3 +1100,76 @@ Feature: Delta Lake Merge
         | 2  | 2024 | 250   |
         | 3  | 2024 | 700   |
         | 4  | 2024 | 900   |
+
+
+  Rule: MERGE INTO with path-based target (delta.`/path/to/table` syntax)
+    Background:
+      Given variable location for temporary directory merge_path_target
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_merge_path_target
+        """
+      Given final statement
+        """
+        DROP VIEW IF EXISTS src_merge_path_target
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_merge_path_target (
+          id INT,
+          value STRING
+        )
+        USING DELTA LOCATION {{ location.sql }}
+        TBLPROPERTIES ('delta.constraints.positive_id' = 'id > 0')
+        """
+      Given statement
+        """
+        INSERT INTO delta_merge_path_target
+        SELECT * FROM VALUES
+          (1, 'old'),
+          (2, 'keep')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW src_merge_path_target AS
+        SELECT * FROM VALUES
+          (1, 'new'),
+          (3, 'insert')
+        AS src(id, value)
+        """
+
+    Scenario: MERGE INTO with delta.`path` target using a temp view as source
+      Given statement template
+        """
+        MERGE INTO delta.`{{ location.string }}` AS tgt
+        USING src_merge_path_target AS src
+        ON tgt.id = src.id
+        WHEN MATCHED THEN UPDATE SET tgt.value = src.value
+        WHEN NOT MATCHED THEN INSERT *
+        """
+      When query
+        """
+        SELECT id, value FROM delta_merge_path_target ORDER BY id
+        """
+      Then query result ordered
+        | id | value  |
+        | 1  | new    |
+        | 2  | keep   |
+        | 3  | insert |
+
+    Scenario: MERGE INTO with delta.`path` target enforces CHECK constraints
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW src_merge_path_target_bad AS
+        SELECT * FROM VALUES
+          (0, 'bad')
+        AS src(id, value)
+        """
+      When query template
+        """
+        MERGE INTO delta.`{{ location.string }}` AS tgt
+        USING src_merge_path_target_bad AS src
+        ON tgt.id = src.id
+        WHEN NOT MATCHED THEN INSERT *
+        """
+      Then query error DELTA_VIOLATE_CONSTRAINT_WITH_VALUES
diff --git a/python/pysail/tests/spark/delta/features/read_path.feature b/python/pysail/tests/spark/delta/features/read_path.feature
index 92afe924a..9a743472d 100644
--- a/python/pysail/tests/spark/delta/features/read_path.feature
+++ b/python/pysail/tests/spark/delta/features/read_path.feature
@@ -29,6 +29,45 @@ Feature: Delta Lake read path (driver vs metadata-as-data)
         """
       Then query plan matches snapshot
 
+  Rule: EXPLAIN shows timestamp schema adaptation inside the driver file scan
+    Background:
+      Given variable location for temporary directory delta_read_driver_timestamp
+      Given config spark.sql.session.timeZone = America/Los_Angeles
+      Given final statement
+        """
+        DROP TABLE IF EXISTS delta_read_driver_timestamp_path
+        """
+      Given statement template
+        """
+        CREATE TABLE delta_read_driver_timestamp_path (
+          id INT,
+          event_time TIMESTAMP
+        )
+        USING DELTA LOCATION {{ location.sql }}
+        """
+      Given statement
+        """
+        INSERT INTO delta_read_driver_timestamp_path
+        VALUES (1, TIMESTAMP '2024-05-01 12:00:00')
+        """
+
+    Scenario: EXPLAIN SELECT with timestamp uses the standard driver file scan
+      When query
+        """
+        EXPLAIN SELECT * FROM delta_read_driver_timestamp_path
+        """
+      Then query plan matches snapshot
+
+    Scenario: SELECT with timestamp preserves the session-local value
+      When query
+        """
+        SELECT id, CAST(event_time AS STRING) AS event_time
+        FROM delta_read_driver_timestamp_path
+        """
+      Then query result ordered
+        | id | event_time          |
+        | 1  | 2024-05-01 12:00:00 |
+
   Rule: EXPLAIN shows metadata-as-data path when table has metadataAsDataRead option
     Background:
       Given variable location for temporary directory delta_read_metadata
diff --git a/python/pysail/tests/spark/delta/test_delta_checkpoint.py b/python/pysail/tests/spark/delta/test_delta_checkpoint.py
index 5b1e87921..637e51d15 100644
--- a/python/pysail/tests/spark/delta/test_delta_checkpoint.py
+++ b/python/pysail/tests/spark/delta/test_delta_checkpoint.py
@@ -3,7 +3,14 @@
 import json
 from typing import TYPE_CHECKING
 
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.parquet as pq
+import pytest
 from pyspark.sql import Row
+from pyspark.sql.types import LongType, StringType, StructField, StructType
+
+from pysail.testing.spark.utils.sql import escape_sql_string_literal
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -20,6 +27,120 @@ def _read_first_commit_metadata(table_path: Path) -> dict:
     raise AssertionError(msg)
 
 
+def _checkpoint_file(table_path: Path, version: int) -> Path:
+    matches = sorted((table_path / "_delta_log").glob(f"{version:020d}.checkpoint*.parquet"))
+    assert len(matches) == 1, f"expected one checkpoint for version {version}, found {matches}"
+    return matches[0]
+
+
+def _write_checkpoint_replacement(checkpoint: Path, table: pa.Table) -> None:
+    replacement = checkpoint.with_name(f"{checkpoint.stem}.replacement.parquet")
+    pq.write_table(table, replacement)
+    replacement.replace(checkpoint)
+
+
+def _drop_null_checkpoint_action(checkpoint: Path, action_name: str) -> None:
+    table = pq.read_table(checkpoint)
+    action = table.column(action_name)
+    assert action.null_count == table.num_rows
+    _write_checkpoint_replacement(checkpoint, table.drop([action_name]))
+
+
+def _drop_null_checkpoint_add_field(checkpoint: Path, field_name: str) -> None:
+    table = pq.read_table(checkpoint)
+    add_index = table.schema.get_field_index("add")
+    add = table.column(add_index).combine_chunks()
+    field_index = add.type.get_field_index(field_name)
+    assert field_index >= 0
+    assert add.field(field_index).null_count == len(add)
+
+    fields = [field for field in add.type if field.name != field_name]
+    columns = [add.field(index) for index, field in enumerate(add.type) if field.name != field_name]
+    replacement_add = pa.StructArray.from_arrays(columns, fields=fields, mask=pc.is_null(add))
+    table_columns = [table.column(index) for index in range(table.num_columns)]
+    table_columns[add_index] = pa.chunked_array([replacement_add])
+    table_fields = list(table.schema)
+    table_fields[add_index] = pa.field("add", replacement_add.type, nullable=True)
+    _write_checkpoint_replacement(
+        checkpoint,
+        pa.Table.from_arrays(
+            table_columns,
+            schema=pa.schema(table_fields, metadata=table.schema.metadata),
+        ),
+    )
+
+
+def _require_legacy_partition_values_parsed(checkpoint: Path) -> None:
+    table = pq.read_table(checkpoint)
+    add_index = table.schema.get_field_index("add")
+    add = table.column(add_index).combine_chunks()
+    parsed_index = add.type.get_field_index("partitionValues_parsed")
+    assert parsed_index >= 0
+
+    fields = list(add.type)
+    parsed_field = fields[parsed_index]
+    fields[parsed_index] = pa.field(
+        parsed_field.name,
+        parsed_field.type,
+        nullable=False,
+        metadata=parsed_field.metadata,
+    )
+    columns = [add.field(index) for index in range(len(add.type))]
+    parsed = columns[parsed_index]
+    columns[parsed_index] = pa.array(
+        [{} if value is None else value for value in parsed.to_pylist()],
+        type=parsed.type,
+    )
+    replacement_add = pa.StructArray.from_arrays(columns, fields=fields, mask=pc.is_null(add))
+
+    table_columns = [table.column(index) for index in range(table.num_columns)]
+    table_columns[add_index] = pa.chunked_array([replacement_add])
+    table_fields = list(table.schema)
+    table_fields[add_index] = pa.field("add", replacement_add.type, nullable=True)
+    _write_checkpoint_replacement(
+        checkpoint,
+        pa.Table.from_arrays(
+            table_columns,
+            schema=pa.schema(table_fields, metadata=table.schema.metadata),
+        ),
+    )
+    assert not pq.read_schema(checkpoint).field("add").type.field("partitionValues_parsed").nullable
+
+
+def _write_partitioned_history_through_json_commit(spark, table_path: Path) -> None:
+    initial = spark.createDataFrame(
+        [
+            Row(id=1, category="A", value=10),
+            Row(id=2, category="B", value=20),
+        ]
+    )
+    (
+        initial.write.format("delta")
+        .mode("overwrite")
+        .option("delta.checkpointInterval", "2")
+        .option("delta.checkpoint.writeStatsAsStruct", "true")
+        .partitionBy("category")
+        .save(str(table_path))
+    )
+
+    for row in [
+        Row(id=3, category="A", value=30),
+        Row(id=4, category="B", value=40),
+        Row(id=5, category="C", value=50),
+    ]:
+        spark.createDataFrame([row]).write.format("delta").mode("append").save(str(table_path))
+
+    _checkpoint_file(table_path, 2)
+    assert (table_path / "_delta_log" / "00000000000000000003.json").is_file()
+
+
+def _assert_struct_tree_nullable(field: pa.Field, path: str) -> None:
+    assert field.nullable, f"checkpoint field {path} must be nullable"
+    if pa.types.is_struct(field.type):
+        for child in field.type:
+            _assert_struct_tree_nullable(child, f"{path}.{child.name}")
+
+
 def test_delta_write_to_table_properties_materialize_metadata(spark, tmp_path: Path):
     """Test that .writeTo().tableProperty() materializes Delta table properties on first commit.
 
@@ -62,3 +183,277 @@ def test_delta_write_option_routes_table_property_to_metadata(spark, tmp_path: P
 
     metadata = _read_first_commit_metadata(base)
     assert metadata.get("configuration", {}).get("delta.checkpointInterval") == "3"
+
+
+def test_partitioned_overwrite_after_checkpoint_and_json_commit(spark, tmp_path: Path):
+    table_path = tmp_path / "delta_checkpoint_partitioned_overwrite"
+
+    def overwrite(version: int) -> None:
+        row = Row(id=version, category=f"category-{version % 2}", value=version * 10)
+        writer = spark.createDataFrame([row]).write.format("delta").mode("overwrite").partitionBy("category")
+        if version == 0:
+            writer = writer.option("delta.checkpointInterval", "2").option(
+                "delta.checkpoint.writeStatsAsStruct", "true"
+            )
+        writer.save(str(table_path))
+
+    for version in range(4):
+        overwrite(version)
+
+    _checkpoint_file(table_path, 2)
+    assert (table_path / "_delta_log" / "00000000000000000003.json").is_file()
+
+    overwrite(4)
+
+    rows = spark.read.format("delta").load(str(table_path)).select("id", "category", "value").collect()
+    assert rows == [Row(id=4, category="category-0", value=40)]
+
+
+@pytest.mark.parametrize("mutation", ["replace_where", "delete"])
+def test_partitioned_mutation_after_checkpoint_and_json_commit(spark, tmp_path: Path, mutation: str):
+    table_path = tmp_path / f"delta_checkpoint_{mutation}"
+    table_name = f"delta_checkpoint_{mutation}"
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+    try:
+        _write_partitioned_history_through_json_commit(spark, table_path)
+
+        if mutation == "replace_where":
+            replacement = spark.createDataFrame([Row(id=10, category="A", value=100)])
+            (
+                replacement.write.format("delta")
+                .mode("overwrite")
+                .option("replaceWhere", "category = 'A'")
+                .save(str(table_path))
+            )
+            expected_ids = [2, 4, 5, 10]
+        else:
+            location = escape_sql_string_literal(str(table_path))
+            spark.sql(f"CREATE TABLE {table_name} USING DELTA LOCATION '{location}'").collect()
+            spark.sql(f"DELETE FROM {table_name} WHERE category = 'A'")  # noqa: S608
+            expected_ids = [2, 4, 5]
+
+        actual_ids = [row.id for row in spark.read.format("delta").load(str(table_path)).orderBy("id").collect()]
+        assert actual_ids == expected_ids
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+
+def test_checkpoint_parsed_fields_are_recursively_nullable(spark, tmp_path: Path):
+    table_path = tmp_path / "delta_checkpoint_nullable_parsed_fields"
+    table_schema = StructType(
+        [
+            StructField("id", LongType(), nullable=False),
+            StructField("category", StringType(), nullable=False),
+            StructField("value", LongType(), nullable=False),
+        ]
+    )
+
+    for version in range(3):
+        frame = spark.createDataFrame(
+            [(version, f"category-{version % 2}", version * 10)],
+            schema=table_schema,
+        )
+        writer = frame.write.format("delta").mode("overwrite" if version == 0 else "append")
+        if version == 0:
+            writer = (
+                writer.option("delta.checkpointInterval", "2")
+                .option("delta.checkpoint.writeStatsAsStruct", "true")
+                .partitionBy("category")
+            )
+        writer.save(str(table_path))
+
+    checkpoint = pq.read_table(_checkpoint_file(table_path, 2))
+    add_field = checkpoint.schema.field("add")
+    stats_parsed = add_field.type.field("stats_parsed")
+    partition_values_parsed = add_field.type.field("partitionValues_parsed")
+
+    _assert_struct_tree_nullable(stats_parsed, "add.stats_parsed")
+    _assert_struct_tree_nullable(partition_values_parsed, "add.partitionValues_parsed")
+
+    add_array = checkpoint.column("add").combine_chunks()
+    partition_values_array = add_array.field(add_array.type.get_field_index("partitionValues_parsed"))
+    assert add_array.null_count > 0
+    assert partition_values_array.null_count >= add_array.null_count
+
+
+def test_legacy_nonnullable_parsed_checkpoint_replays_json_tail(spark, tmp_path: Path):
+    table_path = tmp_path / "delta_checkpoint_legacy_nonnullable_parsed"
+    _write_partitioned_history_through_json_commit(spark, table_path)
+
+    _require_legacy_partition_values_parsed(_checkpoint_file(table_path, 2))
+
+    replayed_ids = [
+        row.id
+        for row in (
+            spark.read.format("delta")
+            .option("metadataAsDataRead", "true")
+            .load(str(table_path))
+            .orderBy("id")
+            .collect()
+        )
+    ]
+    assert replayed_ids == [1, 2, 3, 4, 5]
+
+    replacement = spark.createDataFrame([Row(id=10, category="A", value=100)])
+    (replacement.write.format("delta").mode("overwrite").option("replaceWhere", "category = 'A'").save(str(table_path)))
+    actual_ids = [row.id for row in spark.read.format("delta").load(str(table_path)).orderBy("id").collect()]
+    assert actual_ids == [2, 4, 5, 10]
+
+
+@pytest.mark.parametrize("omitted_action", ["add", "remove"])
+def test_sparse_checkpoint_action_schema_replays_json_tail(
+    spark,
+    tmp_path: Path,
+    omitted_action: str,
+):
+    table_path = tmp_path / f"delta_checkpoint_missing_{omitted_action}"
+    table_name = f"delta_checkpoint_missing_{omitted_action}"
+    location = escape_sql_string_literal(str(table_path))
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (id BIGINT, category STRING)
+            USING DELTA
+            PARTITIONED BY (category)
+            LOCATION '{location}'
+            TBLPROPERTIES ('delta.checkpointInterval' = '2')
+            """
+        ).collect()
+        if omitted_action == "add":
+            spark.sql(f"INSERT INTO {table_name} VALUES (1, 'A'), (2, 'B')")  # noqa: S608
+            spark.sql(f"DELETE FROM {table_name} WHERE TRUE")  # noqa: S608
+            spark.sql(f"INSERT INTO {table_name} VALUES (3, 'C')")  # noqa: S608
+            expected_ids = [3]
+        else:
+            spark.sql(f"INSERT INTO {table_name} VALUES (1, 'A')")  # noqa: S608
+            spark.sql(f"INSERT INTO {table_name} VALUES (2, 'B'), (3, 'C')")  # noqa: S608
+            spark.sql(f"DELETE FROM {table_name} WHERE id = 1")  # noqa: S608
+            expected_ids = [2, 3]
+        spark.sql(f"DROP TABLE {table_name}").collect()
+
+        _drop_null_checkpoint_action(_checkpoint_file(table_path, 2), omitted_action)
+
+        driver_ids = [row.id for row in spark.read.format("delta").load(str(table_path)).orderBy("id").collect()]
+        metadata_ids = [
+            row.id
+            for row in (
+                spark.read.format("delta")
+                .option("metadataAsDataRead", "true")
+                .load(str(table_path))
+                .orderBy("id")
+                .collect()
+            )
+        ]
+        assert driver_ids == expected_ids
+        assert metadata_ids == expected_ids
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+
+@pytest.mark.parametrize("write_stats_as_struct", ["false", "true"])
+def test_checkpoint_without_json_stats_supports_filter_and_delete(
+    spark,
+    tmp_path: Path,
+    write_stats_as_struct: str,
+):
+    suffix = write_stats_as_struct
+    table_path = tmp_path / f"delta_checkpoint_without_json_stats_{suffix}"
+    table_name = f"delta_checkpoint_without_json_stats_{suffix}"
+    location = escape_sql_string_literal(str(table_path))
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (id BIGINT, category STRING, value BIGINT)
+            USING DELTA
+            PARTITIONED BY (category)
+            LOCATION '{location}'
+            TBLPROPERTIES (
+              'delta.checkpointInterval' = '2',
+              'delta.checkpoint.writeStatsAsJson' = 'false',
+              'delta.checkpoint.writeStatsAsStruct' = '{suffix}'
+            )
+            """
+        ).collect()
+        spark.sql(f"INSERT INTO {table_name} VALUES (1, 'A', 10)")  # noqa: S608
+        spark.sql(f"INSERT INTO {table_name} VALUES (2, 'B', 20)")  # noqa: S608
+        checkpoint = _checkpoint_file(table_path, 2)
+        add_fields = {field.name for field in pq.read_schema(checkpoint).field("add").type}
+        assert "stats" not in add_fields
+        assert ("stats_parsed" in add_fields) is (write_stats_as_struct == "true")
+        spark.sql(f"INSERT INTO {table_name} VALUES (3, 'C', 30)")  # noqa: S608
+        spark.sql(f"DROP TABLE {table_name}").collect()
+
+        filtered_ids = [
+            row.id
+            for row in (
+                spark.read.format("delta")
+                .option("metadataAsDataRead", "true")
+                .load(str(table_path))
+                .where("value >= 20")
+                .orderBy("id")
+                .collect()
+            )
+        ]
+        assert filtered_ids == [2, 3]
+
+        spark.sql(f"CREATE TABLE {table_name} USING DELTA LOCATION '{location}'").collect()
+        spark.sql(f"DELETE FROM {table_name} WHERE value = 20")  # noqa: S608
+        remaining_ids = [row.id for row in spark.table(table_name).orderBy("id").collect()]
+        assert remaining_ids == [1, 3]
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+
+def test_deletion_vector_transition_after_checkpoint_preserves_json_field(
+    spark,
+    tmp_path: Path,
+):
+    table_path = tmp_path / "delta_checkpoint_deletion_vector_transition"
+    table_name = "delta_checkpoint_deletion_vector_transition"
+    location = escape_sql_string_literal(str(table_path))
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
+
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (id BIGINT, value STRING)
+            USING DELTA
+            LOCATION '{location}'
+            TBLPROPERTIES ('delta.checkpointInterval' = '2')
+            """
+        ).collect()
+        spark.sql(f"INSERT INTO {table_name} VALUES (1, 'A'), (2, 'B'), (3, 'C')")  # noqa: S608
+        spark.sql(f"INSERT INTO {table_name} VALUES (4, 'D')")  # noqa: S608
+        checkpoint = _checkpoint_file(table_path, 2)
+        _drop_null_checkpoint_add_field(checkpoint, "deletionVector")
+
+        spark.sql(
+            f"ALTER TABLE {table_name} SET TBLPROPERTIES "
+            "('delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '10')"
+        ).collect()
+        spark.sql(f"DELETE FROM {table_name} WHERE id = 2")  # noqa: S608
+
+        delete_commit = table_path / "_delta_log" / "00000000000000000004.json"
+        actions = [json.loads(line) for line in delete_commit.read_text(encoding="utf-8").splitlines()]
+        assert any(action.get("add", {}).get("deletionVector") for action in actions)
+
+        driver_ids = [row.id for row in spark.table(table_name).orderBy("id").collect()]
+        metadata_ids = [
+            row.id
+            for row in (
+                spark.read.format("delta")
+                .option("metadataAsDataRead", "true")
+                .load(str(table_path))
+                .orderBy("id")
+                .collect()
+            )
+        ]
+        assert driver_ids == [1, 3, 4]
+        assert metadata_ids == [1, 3, 4]
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}").collect()
diff --git a/python/pysail/tests/spark/delta/test_delta_column_mapping.py b/python/pysail/tests/spark/delta/test_delta_column_mapping.py
index 7e9d12f53..61cd51e1c 100644
--- a/python/pysail/tests/spark/delta/test_delta_column_mapping.py
+++ b/python/pysail/tests/spark/delta/test_delta_column_mapping.py
@@ -2,13 +2,19 @@
 
 import json
 from typing import TYPE_CHECKING
+from urllib.parse import unquote
 
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
 from pyspark.sql import Row
 from pyspark.sql import functions as F  # noqa: N812
 
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from pyspark.sql import SparkSession
+
 
 def _latest_metadata(base: Path) -> dict:
     for log_file in sorted((base / "_delta_log").glob("*.json"), reverse=True):
@@ -30,6 +36,65 @@ def _physical_name_for_column(metadata: dict, column_name: str) -> str:
     raise AssertionError(message)
 
 
+def _latest_added_parquet_files(base: Path) -> list[Path]:
+    for log_file in sorted((base / "_delta_log").glob("*.json"), reverse=True):
+        added = []
+        with log_file.open("r", encoding="utf-8") as f:
+            for line in f:
+                action = json.loads(line)
+                if "add" in action and action["add"]["path"].endswith(".parquet"):
+                    added.append(base / unquote(action["add"]["path"]))
+        if added:
+            return added
+    return []
+
+
+def _assert_parquet_struct_matches_delta(
+    arrow_type: pa.Schema | pa.StructType,
+    delta_type: dict,
+) -> None:
+    for delta_field in delta_type["fields"]:
+        metadata = delta_field["metadata"]
+        physical_name = metadata["delta.columnMapping.physicalName"]
+        arrow_field = arrow_type.field(physical_name)
+        arrow_metadata = {key.decode(): value.decode() for key, value in (arrow_field.metadata or {}).items()}
+        assert arrow_metadata["PARQUET:field_id"] == str(metadata["delta.columnMapping.id"])
+
+        _assert_parquet_type_matches_delta(arrow_field.type, delta_field["type"])
+
+
+def _assert_parquet_type_matches_delta(
+    arrow_type: pa.DataType,
+    delta_type: dict | str,
+) -> None:
+    if not isinstance(delta_type, dict):
+        return
+
+    type_name = delta_type["type"]
+    if type_name == "struct":
+        _assert_parquet_struct_matches_delta(arrow_type, delta_type)
+    elif type_name == "array":
+        _assert_parquet_type_matches_delta(arrow_type.value_type, delta_type["elementType"])
+    elif type_name == "map":
+        _assert_parquet_type_matches_delta(arrow_type.key_type, delta_type["keyType"])
+        _assert_parquet_type_matches_delta(arrow_type.item_type, delta_type["valueType"])
+
+
+def _assert_parquet_files_match_delta_schema(
+    base: Path,
+    *,
+    latest_only: bool = False,
+) -> None:
+    delta_schema = json.loads(_latest_metadata(base)["schemaString"])
+    parquet_files = _latest_added_parquet_files(base) if latest_only else list(base.rglob("*.parquet"))
+    assert parquet_files
+    for parquet_file in parquet_files:
+        _assert_parquet_struct_matches_delta(
+            pq.ParquetFile(parquet_file).schema_arrow,
+            delta_schema,
+        )
+
+
 def test_create_table_with_column_mapping_name(spark, tmp_path: Path):
     base = tmp_path / "delta_cm_name"
     df = spark.createDataFrame(
@@ -119,6 +184,27 @@ def test_create_and_append_with_column_mapping_id(spark, tmp_path: Path):
     assert int(config["delta.columnMapping.maxColumnId"]) >= 2  # noqa: PLR2004
 
 
+@pytest.mark.parametrize("mapping_mode", ["name", "id"])
+def test_scalar_column_mapping_read_does_not_expose_parquet_field_ids(
+    spark: SparkSession,
+    tmp_path: Path,
+    mapping_mode: str,
+):
+    source_path = tmp_path / f"delta_cm_scalar_source_{mapping_mode}"
+    source = spark.createDataFrame([Row(id=1, label="a")])
+    (
+        source.write.format("delta")
+        .mode("overwrite")
+        .option("delta.columnMapping.mode", mapping_mode)
+        .save(str(source_path))
+    )
+
+    loaded_schema = spark.read.format("delta").load(str(source_path)).schema
+    for field in loaded_schema.fields:
+        assert "PARQUET:field_id" not in field.metadata
+        assert "parquet.field.id" not in field.metadata
+
+
 def test_merge_schema_with_column_mapping_name(spark, tmp_path: Path):
     base = tmp_path / "delta_cm_merge_name"
 
@@ -186,16 +272,104 @@ def test_merge_nested_struct_in_name_mode(spark, tmp_path: Path):
     ]
 
 
+def test_column_mapping_nested_struct_round_trip(spark, tmp_path: Path):
+    source_path = tmp_path / "delta_cm_nested_round_trip_source"
+    output_path = tmp_path / "delta_cm_nested_round_trip_output"
+    source_rows = [
+        Row(
+            id=1,
+            code="USD",
+            original_details=Row(amount=10, active=True),
+        )
+    ]
+
+    (
+        spark.createDataFrame(source_rows)
+        .write.format("delta")
+        .mode("overwrite")
+        .option("delta.columnMapping.mode", "name")
+        .save(str(source_path))
+    )
+
+    loaded = spark.read.format("delta").load(str(source_path))
+    assert [row.asDict(recursive=True) for row in loaded.collect()] == [
+        {
+            "id": 1,
+            "code": "USD",
+            "original_details": {"amount": 10, "active": True},
+        }
+    ]
+    assert loaded.select("original_details.amount").collect() == [Row(amount=10)]
+    _assert_parquet_files_match_delta_schema(source_path)
+
+    rebuilt = loaded.select(F.struct(F.col("id"), F.col("code")).alias("details"))
+    (rebuilt.write.format("delta").mode("overwrite").option("delta.columnMapping.mode", "name").save(str(output_path)))
+
+    result = spark.read.format("delta").load(str(output_path))
+    assert [row.asDict(recursive=True) for row in result.collect()] == [{"details": {"id": 1, "code": "USD"}}]
+    _assert_parquet_files_match_delta_schema(output_path)
+    target_schema = _latest_metadata(output_path)["schemaString"]
+    assert "PARQUET:field_id" not in target_schema
+    assert "parquet.field.id" not in target_schema
+
+
+def test_deep_nested_predicate_with_explicit_schema(spark, tmp_path: Path):
+    base = tmp_path / "delta_cm_deep_predicate"
+    rows = [
+        Row(id=1, payload=Row(level=Row(value=10))),
+        Row(id=2, payload=Row(level=Row(value=20))),
+    ]
+    source = spark.createDataFrame(rows)
+    (source.write.format("delta").mode("overwrite").option("delta.columnMapping.mode", "name").save(str(base)))
+
+    loaded = spark.read.schema(source.schema).format("delta").load(str(base))
+    filtered = loaded.where(F.col("payload.level.value") > F.lit(10)).collect()
+
+    assert [row.asDict(recursive=True) for row in filtered] == [{"id": 2, "payload": {"level": {"value": 20}}}]
+
+
+def test_nested_mapping_with_metadata_as_data_read(spark, tmp_path: Path):
+    base = tmp_path / "delta_cm_metadata_as_data"
+    rows = [
+        Row(id=1, payload=Row(level=Row(value=10))),
+        Row(id=2, payload=Row(level=Row(value=20))),
+    ]
+    source = spark.createDataFrame(rows)
+    (source.write.format("delta").mode("overwrite").option("delta.columnMapping.mode", "name").save(str(base)))
+
+    loaded = spark.read.format("delta").option("metadataAsDataRead", "true").load(str(base)).orderBy("id")
+
+    assert [row.asDict(recursive=True) for row in loaded.collect()] == [
+        {"id": 1, "payload": {"level": {"value": 10}}},
+        {"id": 2, "payload": {"level": {"value": 20}}},
+    ]
+
+
 def test_merge_array_of_struct_in_name_mode(spark, tmp_path: Path):
     base = tmp_path / "delta_cm_array_struct"
     df = spark.createDataFrame([Row(events=[Row(ts=1)])])
-    df.write.format("delta").mode("overwrite").option("column_mapping_mode", "name").save(str(base))
+    (df.write.format("delta").mode("overwrite").option("delta.columnMapping.mode", "name").save(str(base)))
 
     df2 = spark.createDataFrame([Row(events=[Row(ts=2, kind="x")])])
     df2.write.format("delta").mode("append").option("mergeSchema", "true").save(str(base))
 
     rows = [r.asDict(recursive=True) for r in spark.read.format("delta").load(str(base)).collect()]
-    assert any("kind" in ev for row in rows for ev in row.get("events", []) or [])
+    assert {"ts": 2, "kind": "x"} in [event for row in rows for event in row["events"]]
+    _assert_parquet_files_match_delta_schema(base, latest_only=True)
+
+
+def test_merge_schema_after_consecutive_arrays_writes_mapped_parquet_fields(
+    spark: SparkSession,
+    tmp_path: Path,
+):
+    base = tmp_path / "delta_cm_consecutive_arrays"
+    initial = spark.createDataFrame([Row(matrix=[[Row(value=1)]])])
+    (initial.write.format("delta").mode("overwrite").option("delta.columnMapping.mode", "name").save(str(base)))
+
+    appended = spark.createDataFrame([Row(matrix=[[Row(value=2)]], label="new")])
+    appended.write.format("delta").mode("append").option("mergeSchema", "true").save(str(base))
+
+    _assert_parquet_files_match_delta_schema(base, latest_only=True)
 
 
 def test_add_new_array_struct_field(spark, tmp_path: Path):
@@ -213,8 +387,8 @@ def test_add_new_array_struct_field(spark, tmp_path: Path):
     assert rows[1]["id"] == 2  # noqa: PLR2004
     assert isinstance(rows[1]["items"], list)
     assert len(rows[1]["items"]) == 1
-    # Do not assert nested field logical name to avoid coupling to physicalName mapping
-    assert list(rows[1]["items"][0].values()) == [10]
+    assert rows[1]["items"][0] == {"a": 10}
+    _assert_parquet_files_match_delta_schema(base, latest_only=True)
 
 
 def test_merge_map_value_struct(spark, tmp_path: Path):
@@ -225,8 +399,9 @@ def test_merge_map_value_struct(spark, tmp_path: Path):
     df2 = spark.createDataFrame([Row(attrs={"k": Row(a=2, b=3)})])
     df2.write.format("delta").mode("append").option("mergeSchema", "true").save(str(base))
 
-    row = spark.read.format("delta").load(str(base)).collect()[0].asDict(recursive=True)
-    assert "b" in row["attrs"]["k"]
+    rows = [row.asDict(recursive=True) for row in spark.read.format("delta").load(str(base)).collect()]
+    assert {"k": {"a": 2, "b": 3}} in [row["attrs"] for row in rows]
+    _assert_parquet_files_match_delta_schema(base, latest_only=True)
 
 
 def test_partitioned_table_with_column_mapping_name(spark, tmp_path: Path):
@@ -378,3 +553,190 @@ def test_column_mapping_supports_special_characters_in_column_names(spark, tmp_p
 
     projected = out.selectExpr("`first.name`", "`name with space`", "`a,b`").collect()
     assert [row.asDict() for row in projected] == rows
+
+
+RENAMED_EXPECTED_ROWS = [
+    {"id": 1, "code": "alpha", "details": {"total": 10, "active": True}},
+    {"id": 2, "code": "beta", "details": {"total": 20, "active": False}},
+    {"id": 3, "code": "gamma", "details": {"total": 30, "active": True}},
+]
+
+
+def _collect_rows(df) -> list[dict]:
+    return sorted(
+        (row.asDict(recursive=True) for row in df.collect()),
+        key=lambda row: row["id"],
+    )
+
+
+def _rename_schema(schema: dict) -> dict:
+    renamed = json.loads(json.dumps(schema))
+    for field in renamed["fields"]:
+        if field["name"] == "label":
+            field["name"] = "code"
+        if field["name"] == "details":
+            for child in field["type"]["fields"]:
+                if child["name"] == "amount":
+                    child["name"] = "total"
+    return renamed
+
+
+def _append_metadata_only_rename(table_path: Path) -> None:
+    metadata = _latest_metadata(table_path)
+    original_schema = json.loads(metadata["schemaString"])
+    renamed_schema = _rename_schema(original_schema)
+
+    original_fields = {field["name"]: field["metadata"] for field in original_schema["fields"]}
+    renamed_fields = {field["name"]: field["metadata"] for field in renamed_schema["fields"]}
+    assert renamed_fields["code"] == original_fields["label"]
+    original_details = {field["name"]: field["metadata"] for field in original_schema["fields"][2]["type"]["fields"]}
+    renamed_details = {field["name"]: field["metadata"] for field in renamed_schema["fields"][2]["type"]["fields"]}
+    assert renamed_details["total"] == original_details["amount"]
+
+    log_dir = table_path / "_delta_log"
+    version = max(int(path.stem) for path in log_dir.glob("*.json")) + 1
+    renamed_metadata = {
+        **metadata,
+        "schemaString": json.dumps(renamed_schema, separators=(",", ":")),
+    }
+    actions = [
+        {
+            "commitInfo": {
+                "operation": "RENAME COLUMN",
+                "operationParameters": {
+                    "oldColumnPath": "label,details.amount",
+                    "newColumnPath": "code,details.total",
+                },
+                "readVersion": version - 1,
+                "isBlindAppend": True,
+            }
+        },
+        {"metaData": renamed_metadata},
+    ]
+    log_file = log_dir / f"{version:020}.json"
+    log_file.write_text(
+        "".join(f"{json.dumps(action, separators=(',', ':'))}\n" for action in actions),
+        encoding="utf-8",
+    )
+
+
+def _change_nested_physical_name(table_path: Path) -> None:
+    metadata = _latest_metadata(table_path)
+    schema = json.loads(metadata["schemaString"])
+    details = next(field for field in schema["fields"] if field["name"] == "details")
+    amount = next(field for field in details["type"]["fields"] if field["name"] == "amount")
+    amount["metadata"]["delta.columnMapping.physicalName"] = "missing-physical-name"
+
+    log_dir = table_path / "_delta_log"
+    version = max(int(path.stem) for path in log_dir.glob("*.json")) + 1
+    changed_metadata = {
+        **metadata,
+        "schemaString": json.dumps(schema, separators=(",", ":")),
+    }
+    actions = [
+        {
+            "commitInfo": {
+                "operation": "SET TBLPROPERTIES",
+                "readVersion": version - 1,
+                "isBlindAppend": True,
+            }
+        },
+        {"metaData": changed_metadata},
+    ]
+    (log_dir / f"{version:020}.json").write_text(
+        "".join(f"{json.dumps(action, separators=(',', ':'))}\n" for action in actions),
+        encoding="utf-8",
+    )
+
+
+@pytest.fixture
+def renamed_table(spark: SparkSession, tmp_path: Path) -> Path:
+    table_path = tmp_path / "renamed_column_mapping"
+    initial_rows = [
+        Row(id=1, label="alpha", details=Row(amount=10, active=True)),
+        Row(id=2, label="beta", details=Row(amount=20, active=False)),
+    ]
+    (
+        spark.createDataFrame(initial_rows)
+        .write.format("delta")
+        .mode("overwrite")
+        .option("delta.columnMapping.mode", "name")
+        .save(str(table_path))
+    )
+    _append_metadata_only_rename(table_path)
+
+    appended = [Row(id=3, code="gamma", details=Row(total=30, active=True))]
+    spark.createDataFrame(appended).write.format("delta").mode("append").save(str(table_path))
+    return table_path
+
+
+def test_reads_files_written_before_and_after_rename(
+    spark: SparkSession,
+    renamed_table: Path,
+):
+    df = spark.read.format("delta").load(str(renamed_table))
+
+    assert _collect_rows(df) == RENAMED_EXPECTED_ROWS
+    assert sorted(row.total for row in df.select("details.total").collect()) == [
+        10,
+        20,
+        30,
+    ]
+
+
+def test_append_preserves_renamed_nested_mapping(
+    spark: SparkSession,
+    renamed_table: Path,
+):
+    row = [Row(id=4, code="delta", details=Row(total=40, active=False))]
+    spark.createDataFrame(row).write.format("delta").mode("append").save(str(renamed_table))
+
+    assert _collect_rows(spark.read.format("delta").load(str(renamed_table))) == [
+        *RENAMED_EXPECTED_ROWS,
+        {"id": 4, "code": "delta", "details": {"total": 40, "active": False}},
+    ]
+    schema_string = _latest_metadata(renamed_table)["schemaString"]
+    assert "PARQUET:field_id" not in schema_string
+    assert "parquet.field.id" not in schema_string
+
+
+def test_overwrite_preserves_renamed_nested_mapping(
+    spark: SparkSession,
+    renamed_table: Path,
+):
+    original_schema = _latest_metadata(renamed_table)["schemaString"]
+    loaded = spark.read.format("delta").load(str(renamed_table))
+
+    loaded.where("id < 3").write.format("delta").mode("overwrite").save(str(renamed_table))
+
+    assert _collect_rows(spark.read.format("delta").load(str(renamed_table))) == RENAMED_EXPECTED_ROWS[:2]
+    assert _latest_metadata(renamed_table)["schemaString"] == original_schema
+
+
+@pytest.mark.parametrize(
+    ("mode", "expected"),
+    [
+        ("name", None),
+        ("id", 10),
+    ],
+)
+def test_nested_resolution_uses_only_the_configured_identity(
+    spark: SparkSession,
+    tmp_path: Path,
+    mode: str,
+    expected: int | None,
+):
+    table_path = tmp_path / f"nested_resolution_{mode}"
+    rows = [Row(id=1, details=Row(amount=10))]
+    (
+        spark.createDataFrame(rows)
+        .write.format("delta")
+        .mode("overwrite")
+        .option("delta.columnMapping.mode", mode)
+        .save(str(table_path))
+    )
+    _change_nested_physical_name(table_path)
+
+    row = spark.read.format("delta").load(str(table_path)).collect()[0]
+
+    assert row.details.amount == expected
diff --git a/python/pysail/tests/spark/delta/test_delta_column_mapping_delta_spark.py b/python/pysail/tests/spark/delta/test_delta_column_mapping_delta_spark.py
new file mode 100644
index 000000000..9c9987944
--- /dev/null
+++ b/python/pysail/tests/spark/delta/test_delta_column_mapping_delta_spark.py
@@ -0,0 +1,157 @@
+# ruff: noqa: S608
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from pyspark.sql import Row
+from pyspark.sql.types import BooleanType, IntegerType, StructField, StructType
+
+from pysail.testing.spark.utils.sql import escape_sql_identifier
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from pyspark.sql import SparkSession
+
+pytestmark = pytest.mark.integration
+
+
+def _nested_schema(value_field: str) -> StructType:
+    return StructType(
+        [
+            StructField("id", IntegerType(), nullable=True),
+            StructField(
+                "details",
+                StructType(
+                    [
+                        StructField(value_field, IntegerType(), nullable=True),
+                        StructField("active", BooleanType(), nullable=True),
+                    ]
+                ),
+                nullable=True,
+            ),
+        ]
+    )
+
+
+def test_delta_spark_nested_rename_is_readable_and_appendable_by_sail(
+    delta_jvm_spark: SparkSession,
+    spark: SparkSession,
+    tmp_path: Path,
+) -> None:
+    table_path = tmp_path / "delta_spark_nested_mapping"
+    table_identifier = f"delta.`{escape_sql_identifier(str(table_path))}`"
+    delta_jvm_spark.sql(
+        f"""
+        CREATE TABLE {table_identifier} (
+            id INT,
+            details STRUCT
+        )
+        USING DELTA
+        TBLPROPERTIES ('delta.columnMapping.mode' = 'name')
+        """
+    )
+    delta_jvm_spark.sql(
+        f"""
+        INSERT INTO {table_identifier}
+        VALUES
+            (1, named_struct('amount', 10, 'active', true)),
+            (2, named_struct('amount', 20, 'active', false))
+        """
+    )
+    data_files_before_rename = sorted(table_path.rglob("*.parquet"))
+
+    delta_jvm_spark.sql(f"ALTER TABLE {table_identifier} RENAME COLUMN details.amount TO total")
+    assert sorted(table_path.rglob("*.parquet")) == data_files_before_rename
+
+    delta_jvm_spark.sql(
+        f"""
+        INSERT INTO {table_identifier}
+        VALUES (3, named_struct('total', 30, 'active', true))
+        """
+    )
+
+    frame = spark.read.format("delta").load(str(table_path))
+    rows = frame.selectExpr("id", "details.total AS total", "details.active AS active").orderBy("id").collect()
+    assert [(row.id, row.total, row.active) for row in rows] == [
+        (1, 10, True),
+        (2, 20, False),
+        (3, 30, True),
+    ]
+    assert [row.id for row in frame.where("details.total >= 20").select("id").orderBy("id").collect()] == [2, 3]
+
+    append = spark.createDataFrame(
+        [Row(id=4, details=Row(total=40, active=False))],
+        schema=_nested_schema("total"),
+    )
+    append.write.format("delta").mode("append").save(str(table_path))
+
+    delta_jvm_spark.catalog.refreshByPath(str(table_path))
+    rows = delta_jvm_spark.sql(
+        f"""
+        SELECT id, details.total AS total, details.active AS active
+        FROM {table_identifier}
+        ORDER BY id
+        """
+    ).collect()
+    assert [(row.id, row.total, row.active) for row in rows] == [
+        (1, 10, True),
+        (2, 20, False),
+        (3, 30, True),
+        (4, 40, False),
+    ]
+
+
+def test_sail_nested_mapping_is_readable_and_appendable_by_delta_spark(
+    delta_jvm_spark: SparkSession,
+    spark: SparkSession,
+    tmp_path: Path,
+) -> None:
+    table_path = tmp_path / "sail_nested_mapping"
+    table_identifier = f"delta.`{escape_sql_identifier(str(table_path))}`"
+    source = spark.createDataFrame(
+        [
+            Row(id=1, details=Row(amount=10, active=True)),
+            Row(id=2, details=Row(amount=20, active=False)),
+        ],
+        schema=_nested_schema("amount"),
+    )
+    (source.write.format("delta").mode("overwrite").option("delta.columnMapping.mode", "name").save(str(table_path)))
+
+    rows = delta_jvm_spark.sql(
+        f"""
+        SELECT id, details.amount AS amount, details.active AS active
+        FROM {table_identifier}
+        ORDER BY id
+        """
+    ).collect()
+    assert [(row.id, row.amount, row.active) for row in rows] == [
+        (1, 10, True),
+        (2, 20, False),
+    ]
+    filtered = delta_jvm_spark.sql(
+        f"""
+        SELECT id
+        FROM {table_identifier}
+        WHERE details.amount >= 20
+        ORDER BY id
+        """
+    ).collect()
+    assert [row.id for row in filtered] == [2]
+
+    delta_jvm_spark.sql(
+        f"""
+        INSERT INTO {table_identifier}
+        VALUES (3, named_struct('amount', 30, 'active', true))
+        """
+    )
+
+    frame = spark.read.format("delta").load(str(table_path))
+    rows = frame.selectExpr("id", "details.amount AS amount", "details.active AS active").orderBy("id").collect()
+    assert [(row.id, row.amount, row.active) for row in rows] == [
+        (1, 10, True),
+        (2, 20, False),
+        (3, 30, True),
+    ]
diff --git a/python/pysail/tests/spark/delta/test_delta_merge_into_dataframe.py b/python/pysail/tests/spark/delta/test_delta_merge_into_dataframe.py
index 11637cf9b..f92712e87 100644
--- a/python/pysail/tests/spark/delta/test_delta_merge_into_dataframe.py
+++ b/python/pysail/tests/spark/delta/test_delta_merge_into_dataframe.py
@@ -4,6 +4,7 @@
 from pyspark.sql import functions as F  # noqa: N812
 from pyspark.sql.dataframe import DataFrame as SparkDataFrame
 
+from pysail.testing.spark.utils.common import pyspark_version
 from pysail.testing.spark.utils.sql import escape_sql_string_literal
 
 if not hasattr(SparkDataFrame, "mergeInto"):
@@ -69,3 +70,34 @@ def test_dataframe_merge_into_insert_all_and_delete(spark, tmp_path):
         assert_frame_equal(result, expected, check_dtype=False)
     finally:
         spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+
+
+@pytest.mark.skipif(
+    pyspark_version() < (4,),
+    reason="DataFrame arguments in spark.sql() require Spark 4+",
+)
+def test_merge_into_path_target_with_dataframe_source(spark, tmp_path):
+    table_path = tmp_path / "delta_merge_df_arg_target"
+    table_path_str = str(table_path)
+
+    target_df = spark.createDataFrame([(1, "old"), (2, "keep")], "id INT, value STRING")
+    target_df.write.format("delta").mode("overwrite").save(table_path_str)
+
+    source_df = spark.createDataFrame([(1, "new"), (3, "insert")], "id INT, value STRING")
+
+    spark.sql(
+        f"""
+        MERGE INTO delta.`{table_path_str}` AS tgt
+        USING {{src_df}} AS src
+        ON tgt.id = src.id
+        WHEN MATCHED THEN UPDATE SET tgt.value = src.value
+        WHEN NOT MATCHED THEN INSERT *
+        """,
+        src_df=source_df,
+    )
+
+    result = spark.read.format("delta").load(table_path_str).sort("id").toPandas()
+    expected = pd.DataFrame({"id": [1, 2, 3], "value": ["new", "keep", "insert"]}).astype(
+        {"id": "int32", "value": "string"}
+    )
+    assert_frame_equal(result, expected, check_dtype=False)
diff --git a/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml b/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml
new file mode 100644
index 000000000..d1400dbe4
--- /dev/null
+++ b/python/pysail/tests/spark/execution/__snapshots__/features/explain.yaml
@@ -0,0 +1,360 @@
+# serializer version: 1
+---
+name: "test_explain_codegen_includes_the_distributed_execution_plan"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: t.#2 AS #4, sum(t.#3) AS total AS #5
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(t.#3)]]
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Projection: t.#2 AS #4, sum(t.#3) AS total AS #5
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(3))
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: t.#2 AS #4, sum(t.#3) AS total AS #5
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Values: (Int32(1), Int32(2)), (Int32(1), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: t.#2 AS #4, sum(t.#3) AS total AS #5
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Values: (Int32(1), Int32(2)), (Int32(1), Int32(3))
+  
+    initial_physical_plan:
+    ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+      AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+        AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5], schema=[#4:Int32;N, #5:Int64;N]
+      AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3):Int64;N]
+        AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3)[sum]:Int64;N]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+      AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+        RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#4@0 as k, #5@1 as total]
+      ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#4@0 as k, #5@1 as total]
+      ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+      ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#4@0 as k, #5@1 as total]
+      ProjectionExec: expr=[#2@0 as #4, sum(t.#3)@1 as #5]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          StageInputExec: input=0, partitioning=Hash([#2@0], 4)
diff --git a/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml b/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml
new file mode 100644
index 000000000..86cca19a0
--- /dev/null
+++ b/python/pysail/tests/spark/execution/__snapshots__/features/lakehouse_commit.yaml
@@ -0,0 +1,914 @@
+# serializer version: 1
+---
+name: "test_delta_commit_runs_on_the_driver_after_parallel_file_writing"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Barrier
+      DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: CASE WHEN Boolean(true) THEN id ELSE Int64(NULL) END AS id
+          Projection: #1 AS id
+            Projection: range.#0 AS #1
+              Projection: range.id AS #0
+                TableScan: range
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Barrier
+      DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: id AS id
+          Projection: #1 AS id
+            Projection: range.#0 AS #1
+              Projection: range.id AS #0
+                TableScan: range
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Barrier
+      DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: #1 AS id
+          Projection: range.id AS #1
+            TableScan: range projection=[id]
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Barrier
+      DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: range.id AS id
+          TableScan: range projection=[id]
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Barrier
+      DeltaWrite: options=DeltaWriteNodeOptions { path: "/distributed_delta_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [] }, OptionList { items: [("path", "/distributed_delta_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_delta_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("/distributed_delta_commit") }, operation: Write, format: Delta, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_delta_commit:Delta:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: range.id AS id
+          TableScan: range projection=[id]
+  
+    initial_physical_plan:
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+      CoalescePartitionsExec
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+          SortExec: expr=[1 ASC], preserve_partitioning=[false]
+            RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4
+              ProjectionExec: expr=[id@0 as id]
+                ProjectionExec: expr=[id@0 as id]
+                  RangeExec
+  
+  
+    initial_physical_plan_with_stats:
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+      CoalescePartitionsExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+          SortExec: expr=[1 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+            RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+              ProjectionExec: expr=[id@0 as id], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+                ProjectionExec: expr=[id@0 as id], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+                  RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+  
+  
+    initial_physical_plan_with_schema:
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/), schema=[count:UInt64;N]
+      CoalescePartitionsExec, schema=[action:Union(Dense, 0: ("add": non-null Struct("path": non-null Utf8, "partitionValues": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": non-null Int64, "modificationTime": non-null Int64, "dataChange": non-null Boolean, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64, "clusteringProvider": Utf8)), 1: ("remove": non-null Struct("path": non-null Utf8, "dataChange": non-null Boolean, "deletionTimestamp": Int64, "extendedFileMetadata": Boolean, "partitionValues": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": Int64, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64)), 2: ("protocol": non-null Struct("minReaderVersion": non-null Int32, "minWriterVersion": non-null Int32, "readerFeatures": List(Utf8, field: 'element'), "writerFeatures": List(Utf8, field: 'element'))), 3: ("metadata": non-null Struct("id": non-null Utf8, "name": Utf8, "description": Utf8, "format": non-null Struct("provider": non-null Utf8, "options": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted)), "schemaString": non-null Utf8, "partitionColumns": non-null List(Utf8, field: 'element'), "createdTime": Int64, "configuration": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted))), 4: ("commit_meta": non-null Struct("commit_row_count": non-null UInt64, "operation_json": Utf8, "operation_metrics_json": Utf8)))]
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/), schema=[action:Union(Dense, 0: ("add": non-null Struct("path": non-null Utf8, "partitionValues": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": non-null Int64, "modificationTime": non-null Int64, "dataChange": non-null Boolean, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64, "clusteringProvider": Utf8)), 1: ("remove": non-null Struct("path": non-null Utf8, "dataChange": non-null Boolean, "deletionTimestamp": Int64, "extendedFileMetadata": Boolean, "partitionValues": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "size": Int64, "stats": Utf8, "tags": Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted), "deletionVector": Struct("storageType": non-null Utf8, "pathOrInlineDv": non-null Utf8, "offset": Int32, "sizeInBytes": non-null Int32, "cardinality": non-null Int64), "baseRowId": Int64, "defaultRowCommitVersion": Int64)), 2: ("protocol": non-null Struct("minReaderVersion": non-null Int32, "minWriterVersion": non-null Int32, "readerFeatures": List(Utf8, field: 'element'), "writerFeatures": List(Utf8, field: 'element'))), 3: ("metadata": non-null Struct("id": non-null Utf8, "name": Utf8, "description": Utf8, "format": non-null Struct("provider": non-null Utf8, "options": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted)), "schemaString": non-null Utf8, "partitionColumns": non-null List(Utf8, field: 'element'), "createdTime": Int64, "configuration": non-null Map("key_value": non-null Struct("key": non-null Utf8, "value": Utf8), unsorted))), 4: ("commit_meta": non-null Struct("commit_row_count": non-null UInt64, "operation_json": Utf8, "operation_metrics_json": Utf8)))]
+          SortExec: expr=[1 ASC], preserve_partitioning=[false], schema=[id:Int64]
+            RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, schema=[id:Int64]
+              ProjectionExec: expr=[id@0 as id], schema=[id:Int64]
+                ProjectionExec: expr=[id@0 as id], schema=[id:Int64]
+                  RangeExec, schema=[id:Int64]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+        CoalescePartitionsExec
+          DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+            SortExec: expr=[1 ASC], preserve_partitioning=[false]
+              RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4
+                ProjectionExec: expr=[id@0 as id]
+                  ProjectionExec: expr=[id@0 as id]
+                    RangeExec
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+        CoalescePartitionsExec
+          DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+            RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+              SortExec: expr=[1 ASC], preserve_partitioning=[false]
+                CoalescePartitionsExec
+                  ProjectionExec: expr=[id@0 as id]
+                    ProjectionExec: expr=[id@0 as id]
+                      RangeExec
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+        CoalescePartitionsExec
+          DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+            ProjectionExec: expr=[id@0 as id]
+              ProjectionExec: expr=[id@0 as id]
+                RangeExec
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+        CoalescePartitionsExec
+          DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+            ProjectionExec: expr=[id@0 as id]
+              RangeExec
+  
+  
+    physical_plan after OutputRequirements:
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+      CoalescePartitionsExec
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+          ProjectionExec: expr=[id@0 as id]
+            RangeExec
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+      CoalescePartitionsExec
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+          RangeExec
+  
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+      CoalescePartitionsExec
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+          RangeExec
+  
+  
+    == Physical Plan ==
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+      CoalescePartitionsExec
+        DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+          RangeExec
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    DeltaWriterExec(table_path=file:///distributed_delta_commit/)
+      RangeExec
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Driver
+    DeltaCommitExec(table_path=file:///distributed_delta_commit/)
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Forward)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_iceberg_commit_runs_on_the_driver_after_parallel_file_writing"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Barrier
+      IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: CASE WHEN Boolean(true) THEN id ELSE Int64(NULL) END AS id
+          Projection: #1 AS id
+            Projection: range.#0 AS #1
+              Projection: range.id AS #0
+                TableScan: range
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Barrier
+      IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: id AS id
+          Projection: #1 AS id
+            Projection: range.#0 AS #1
+              Projection: range.id AS #0
+                TableScan: range
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Barrier
+      IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: #1 AS id
+          Projection: range.id AS #1
+            TableScan: range projection=[id]
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Barrier
+      IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: range.id AS id
+          TableScan: range projection=[id]
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Barrier
+      IcebergWrite: options=IcebergWriteNodeOptions { path: "file:///distributed_iceberg_commit", mode: Append, partition_by: [], bucket_by: None, sort_order: [], options: [TablePropertyList { items: [("metadata_location", "file:///distributed_iceberg_commit/metadata/v1.metadata.json")] }, OptionList { items: [("path", "file:///distributed_iceberg_commit")] }], lakehouse_table: Some(LakehouseExecutionContext { catalog_provider_id: CatalogProviderId("sail"), catalog_table: ["distributed_iceberg_commit"], table_identity: CatalogTableIdentity { table_id: None, table_uri: Some("file:///distributed_iceberg_commit") }, operation: Write, format: Iceberg, authority: CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }, pointer: StorageDiscovery, commit: Filesystem, scan: ClientTableFormat, access_session: None, rest_session: None, versioned_catalog: None, cross_format: None, governance: None, capability_fingerprint: CapabilityFingerprint("sail:distributed_iceberg_commit:Iceberg:CatalogRegistered { lifecycle: External, pointer: StorageDiscovery, commit: Filesystem }") }) }
+        Projection: range.id AS id
+          TableScan: range projection=[id]
+  
+    initial_physical_plan:
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+      IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4
+          ProjectionExec: expr=[id@0 as id]
+            RangeExec
+  
+  
+    initial_physical_plan_with_stats:
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+      IcebergWriterExec(table_path=file:///distributed_iceberg_commit/), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+          ProjectionExec: expr=[id@0 as id], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+            RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+  
+  
+    initial_physical_plan_with_schema:
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/), schema=[count:Int64;N]
+      IcebergWriterExec(table_path=file:///distributed_iceberg_commit/), schema=[action:Union(Dense, 0: ("add": non-null Struct("content": non-null Utf8, "file_path": non-null Utf8, "file_format": non-null Utf8, "partition": non-null List(non-null Union(Dense, 0: ("Boolean": non-null Boolean), 1: ("Int": non-null Int32), 2: ("Long": non-null Int64), 3: ("Float": non-null Float32), 4: ("Double": non-null Float64), 5: ("String": non-null Utf8), 6: ("Int128": non-null Utf8), 7: ("UInt128": non-null Utf8), 8: ("Binary": non-null List(non-null UInt8, field: 'element')), 9: ("Null": non-null Boolean)), field: 'element'), "record_count": non-null UInt64, "file_size_in_bytes": non-null UInt64, "column_sizes": non-null Map("entries": non-null Struct("keys": non-null Int32, "values": non-null UInt64), unsorted), "value_counts": non-null Map("entries": non-null Struct("keys": non-null Int32, "values": non-null UInt64), unsorted), "null_value_counts": non-null Map("entries": non-null Struct("keys": non-null Int32, "values": non-null UInt64), unsorted), "split_offsets": non-null List(non-null Int64, field: 'element'), "equality_ids": non-null List(non-null Int32, field: 'element'), "sort_order_id": Int32, "first_row_id": Int64, "partition_spec_id": non-null Int32, "referenced_data_file": Utf8, "content_offset": Int64, "content_size_in_bytes": Int64)), 1: ("delete": non-null Struct("data_file": non-null Struct("content": non-null Utf8, "file_path": non-null Utf8, "file_format": non-null Utf8, "partition": non-null List(non-null Union(Dense, 0: ("Boolean": non-null Boolean), 1: ("Int": non-null Int32), 2: ("Long": non-null Int64), 3: ("Float": non-null Float32), 4: ("Double": non-null Float64), 5: ("String": non-null Utf8), 6: ("Int128": non-null Utf8), 7: ("UInt128": non-null Utf8), 8: ("Binary": non-null List(non-null UInt8, field: 'element')), 9: ("Null": non-null Boolean)), field: 'element'), "record_count": non-null UInt64, "file_size_in_bytes": non-null UInt64, "column_sizes": non-null Map("entries": non-null Struct("key": non-null Int32, "value": non-null UInt64), unsorted), "value_counts": non-null Map("entries": non-null Struct("key": non-null Int32, "value": non-null UInt64), unsorted), "null_value_counts": non-null Map("entries": non-null Struct("key": non-null Int32, "value": non-null UInt64), unsorted), "split_offsets": non-null List(non-null Int64, field: 'element'), "equality_ids": non-null List(non-null Int32, field: 'element'), "sort_order_id": Int32, "first_row_id": Int64, "partition_spec_id": non-null Int32, "referenced_data_file": Utf8, "content_offset": Int64, "content_size_in_bytes": Int64))), 2: ("commit_meta": non-null Struct("table_uri": non-null Utf8, "row_count": non-null UInt64, "requirements_json": non-null Utf8, "table_properties_json": non-null Utf8, "lakehouse_table_json": Utf8, "schema_json": Utf8, "partition_spec_json": Utf8)))]
+        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4, schema=[id:Int64]
+          ProjectionExec: expr=[id@0 as id], schema=[id:Int64]
+            RangeExec, schema=[id:Int64]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+        IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=4
+            ProjectionExec: expr=[id@0 as id]
+              RangeExec
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+        CoalescePartitionsExec
+          IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+            ProjectionExec: expr=[id@0 as id]
+              RangeExec
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+        CoalescePartitionsExec
+          IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+            RangeExec
+  
+  
+    physical_plan after OutputRequirements:
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+      CoalescePartitionsExec
+        IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+          RangeExec
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+      CoalescePartitionsExec
+        IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+          RangeExec
+  
+  
+    == Physical Plan ==
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+      CoalescePartitionsExec
+        IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+          RangeExec
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    IcebergWriterExec(table_path=file:///distributed_iceberg_commit/)
+      RangeExec
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Driver
+    IcebergCommitExec(table_path=file:///distributed_iceberg_commit/)
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Forward)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=UnknownPartitioning(1)
diff --git a/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml b/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml
new file mode 100644
index 000000000..80a55c1c3
--- /dev/null
+++ b/python/pysail/tests/spark/execution/__snapshots__/features/scalar_subquery.yaml
@@ -0,0 +1,11389 @@
+# serializer version: 1
+---
+name: "test_correlated_scalar_subquery_in_filter"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Filter: outer_t.#3 = ()
+          Subquery:
+            Projection: max(inner_t.#7) AS #8
+              Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]]
+                Filter: inner_t.#6 = outer_ref(outer_t.#2)
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Projection: outer_t.#2, outer_t.#3
+          Filter: outer_t.#3 = __scalar_sq_1.#8
+            Left Join:  Filter: __scalar_sq_1.#6 = outer_t.#2
+              SubqueryAlias: outer_t
+                Projection: #0 AS #2, #1 AS #3
+                  Projection: column1 AS #0, column2 AS #1
+                    Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+              SubqueryAlias: __scalar_sq_1
+                Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+                  Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Projection: outer_t.#2, outer_t.#3
+          Filter: outer_t.#3 = __scalar_sq_1.#8
+            Left Join: outer_t.#2 = __scalar_sq_1.#6
+              SubqueryAlias: outer_t
+                Projection: #0 AS #2, #1 AS #3
+                  Projection: column1 AS #0, column2 AS #1
+                    Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+              SubqueryAlias: __scalar_sq_1
+                Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+                  Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Projection: outer_t.#2, outer_t.#3
+          Filter: outer_t.#3 = __scalar_sq_1.#8
+            Inner Join: outer_t.#2 = __scalar_sq_1.#6
+              SubqueryAlias: outer_t
+                Projection: #0 AS #2, #1 AS #3
+                  Projection: column1 AS #0, column2 AS #1
+                    Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+              SubqueryAlias: __scalar_sq_1
+                Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+                  Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Projection: outer_t.#2, outer_t.#3
+          Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#8
+            SubqueryAlias: outer_t
+              Projection: #0 AS #2, #1 AS #3
+                Projection: column1 AS #0, column2 AS #1
+                  Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+            SubqueryAlias: __scalar_sq_1
+              Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+                Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Projection: outer_t.#2, outer_t.#3
+          Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#8
+            SubqueryAlias: outer_t
+              Projection: #0 AS #2, #1 AS #3
+                Projection: column1 AS #0, column2 AS #1
+                  Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+            SubqueryAlias: __scalar_sq_1
+              Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+                Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7)
+                  Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#8
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#8
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10
+        Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#8
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    initial_physical_plan:
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+        HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+        HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                  DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], schema=[#9:Int32;N, #10:Int32;N]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10], schema=[#9:Int32;N, #10:Int32;N]
+        HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], schema=[#2:Int32;N, #3:Int32;N, #8:Int32;N, #6:Int32;N]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], schema=[#8:Int32;N, #6:Int32;N]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N]
+                  DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+          HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+          HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+        CoalescePartitionsExec
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    SortPreservingMergeExec: [#9@0 ASC]
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+          HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v]
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v]
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#6@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10]
+        HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #8@0)], projection=[#2@0, #3@1]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+              StageInputExec: input=0, partitioning=Hash([#6@0], 4)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Merge)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v]
+      SortPreservingMergeExec: [#9@0 ASC]
+        StageInputExec: input=0, partitioning=Hash([#9@0], 4)
+---
+name: "test_correlated_scalar_subquery_in_projection"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, () AS mx AS #11
+        Subquery:
+          Projection: max(inner_t.#7) AS #8
+            Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]]
+              Filter: inner_t.#6 = outer_ref(outer_t.#2)
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+        SubqueryAlias: outer_t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join:  Filter: __scalar_sq_1.#6 = outer_t.#2
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+              Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+              Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+              Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7)
+                Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #9 ASC NULLS FIRST, #10 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    initial_physical_plan:
+    SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+        HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+        HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                  DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N]
+        HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], schema=[#2:Int32;N, #3:Int32;N, #8:Int32;N, #6:Int32;N]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], schema=[#8:Int32;N, #6:Int32;N]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N]
+                  DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+          HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+          HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[false]
+        CoalescePartitionsExec
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC]
+        SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[(#9@0, asc), (#10@1, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC]
+        SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC]
+      SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+          HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx]
+      SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC]
+        SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx]
+      SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC]
+        SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#6@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[#6@1], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+      AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+        StageInputExec: input=0, partitioning=Hash([#6@0], 4)
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=0, mode=Shuffle), StageInput(stage=2, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    SortExec: expr=[#9@0 ASC, #10@1 ASC], preserve_partitioning=[true]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+        HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+          StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+          StageInputExec: input=1, partitioning=Hash([#6@1], 4)
+  
+    === stage 4 ===
+    inputs=[StageInput(stage=3, mode=Merge)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx]
+      SortPreservingMergeExec: [#9@0 ASC, #10@1 ASC]
+        StageInputExec: input=0, partitioning=Hash([#9@0], 4)
+---
+name: "test_correlated_scalar_subquery_nomatch_result"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, () AS mx AS #11
+        Subquery:
+          Projection: max(inner_t.#7) AS #8
+            Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]]
+              Filter: inner_t.#6 = outer_ref(outer_t.#2)
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2))
+        SubqueryAlias: outer_t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join:  Filter: __scalar_sq_1.#6 = outer_t.#2
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+              Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2))
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+              Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6, __always_true
+              Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7)
+                Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2))
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #9 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #9, outer_t.#3 AS #10, __scalar_sq_1.#8 AS mx AS #11
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) AS #8, inner_t.#6
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2))
+  
+    initial_physical_plan:
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+        HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+        HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                  DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[false], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11], schema=[#9:Int32;N, #10:Int32;N, #11:Int32;N]
+        HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], schema=[#2:Int32;N, #3:Int32;N, #8:Int32;N, #6:Int32;N]
+          ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+          ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6], schema=[#8:Int32;N, #6:Int32;N]
+            AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N]
+              AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N]
+                ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N]
+                  DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+          HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+          HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[false]
+        CoalescePartitionsExec
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[(#9@0, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    SortPreservingMergeExec: [#9@0 ASC]
+      SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+          HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx]
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx]
+      SortPreservingMergeExec: [#9@0 ASC]
+        SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#6@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[#6@1], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[max(inner_t.#7)@1 as #8, #6@0 as #6]
+      AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+        StageInputExec: input=0, partitioning=Hash([#6@0], 4)
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=0, mode=Shuffle), StageInput(stage=2, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    SortExec: expr=[#9@0 ASC], preserve_partitioning=[true]
+      ProjectionExec: expr=[#2@0 as #9, #3@1 as #10, #8@2 as #11]
+        HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #8@2]
+          StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+          StageInputExec: input=1, partitioning=Hash([#6@1], 4)
+  
+    === stage 4 ===
+    inputs=[StageInput(stage=3, mode=Merge)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#9@0 as k, #10@1 as v, #11@2 as mx]
+      SortPreservingMergeExec: [#9@0 ASC]
+        StageInputExec: input=0, partitioning=Hash([#9@0], 4)
+---
+name: "test_correlated_scalar_subquery_with_nested_scalar_in_filter"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Filter: outer_t.#3 = ()
+          Subquery:
+            Projection: max(inner_t.#7) + () AS #11
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: #8 AS #9
+                        Projection: column1 AS #8
+                          Values: (Int32(1))
+              Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]]
+                Filter: inner_t.#6 = outer_ref(outer_t.#2)
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Projection: outer_t.#2, outer_t.#3
+          Filter: outer_t.#3 = __scalar_sq_1.#11
+            Left Join:  Filter: __scalar_sq_1.#6 = outer_t.#2
+              SubqueryAlias: outer_t
+                Projection: #0 AS #2, #1 AS #3
+                  Projection: column1 AS #0, column2 AS #1
+                    Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+              SubqueryAlias: __scalar_sq_1
+                Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+                  Subquery:
+                    Projection: min(u.#9) AS #10
+                      Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                        SubqueryAlias: u
+                          Projection: #8 AS #9
+                            Projection: column1 AS #8
+                              Values: (Int32(1))
+                  Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Projection: outer_t.#2, outer_t.#3
+          Filter: outer_t.#3 = __scalar_sq_1.#11
+            Left Join: outer_t.#2 = __scalar_sq_1.#6
+              SubqueryAlias: outer_t
+                Projection: #0 AS #2, #1 AS #3
+                  Projection: column1 AS #0, column2 AS #1
+                    Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+              SubqueryAlias: __scalar_sq_1
+                Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+                  Subquery:
+                    Projection: min(u.#9) AS #10
+                      Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                        SubqueryAlias: u
+                          Projection: #8 AS #9
+                            Projection: column1 AS #8
+                              Values: (Int32(1))
+                  Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Projection: outer_t.#2, outer_t.#3
+          Filter: outer_t.#3 = __scalar_sq_1.#11
+            Inner Join: outer_t.#2 = __scalar_sq_1.#6
+              SubqueryAlias: outer_t
+                Projection: #0 AS #2, #1 AS #3
+                  Projection: column1 AS #0, column2 AS #1
+                    Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+              SubqueryAlias: __scalar_sq_1
+                Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+                  Subquery:
+                    Projection: min(u.#9) AS #10
+                      Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                        SubqueryAlias: u
+                          Projection: #8 AS #9
+                            Projection: column1 AS #8
+                              Values: (Int32(1))
+                  Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Projection: outer_t.#2, outer_t.#3
+          Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#11
+            SubqueryAlias: outer_t
+              Projection: #0 AS #2, #1 AS #3
+                Projection: column1 AS #0, column2 AS #1
+                  Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+            SubqueryAlias: __scalar_sq_1
+              Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+                Subquery:
+                  Projection: min(u.#9) AS #10
+                    Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                      SubqueryAlias: u
+                        Projection: #8 AS #9
+                          Projection: column1 AS #8
+                            Values: (Int32(1))
+                Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Projection: outer_t.#2, outer_t.#3
+          Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#11
+            SubqueryAlias: outer_t
+              Projection: #0 AS #2, #1 AS #3
+                Projection: column1 AS #0, column2 AS #1
+                  Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+            SubqueryAlias: __scalar_sq_1
+              Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+                Subquery:
+                  Projection: min(u.#9) AS #10
+                    Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                      SubqueryAlias: u
+                        Projection: #8 AS #9
+                          Projection: column1 AS #8
+                            Values: (Int32(1))
+                Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7)
+                  Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                    SubqueryAlias: inner_t
+                      Projection: #4 AS #6, #5 AS #7
+                        Projection: column1 AS #4, column2 AS #5
+                          Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Inner Join: outer_t.#2 = __scalar_sq_1.#6 Filter: outer_t.#3 = __scalar_sq_1.#11
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: column1 AS #9
+                        Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#11
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: column1 AS #9
+                        Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #12 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13
+        Inner Join: outer_t.#2 = __scalar_sq_1.#6, outer_t.#3 = __scalar_sq_1.#11
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(5)), (Int32(2), Int32(1)), (Int32(2), Int32(4))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: column1 AS #9
+                        Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#12@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+          HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(u.#9)@0 as #10]
+        AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      SortExec: expr=[#12@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+          HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                    DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(u.#9)@0 as #10], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #9], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#12:Int32;N, #13:Int32;N]
+      SortExec: expr=[#12@0 ASC], preserve_partitioning=[false], schema=[#12:Int32;N, #13:Int32;N]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13], schema=[#12:Int32;N, #13:Int32;N]
+          HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], schema=[#2:Int32;N, #3:Int32;N, #11:Int32;N, #6:Int32;N]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], schema=[#11:Int32;N, #6:Int32;N]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N]
+                    DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[min(u.#9)@0 as #10], schema=[#10:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], schema=[min(u.#9):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], schema=[min(u.#9)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #9], schema=[#9:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+            HashJoinExec: mode=Auto, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+              ProjectionExec: expr=[column1@0 as #9]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+              ProjectionExec: expr=[column1@0 as #9]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC], preserve_partitioning=[false]
+          CoalescePartitionsExec
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+              ProjectionExec: expr=[column1@0 as #9]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC], preserve_partitioning=[false]
+          CoalescePartitionsExec
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC]
+          SortExec: expr=[#12@0 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC]
+          SortExec: expr=[#12@0 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortPreservingMergeExec: [#12@0 ASC]
+        SortExec: expr=[#12@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(u.#9)@0 as #10]
+        AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+          ProjectionExec: expr=[column1@0 as #9]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#12@0 as k, #13@1 as v]
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC]
+          SortExec: expr=[#12@0 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#12@0 as k, #13@1 as v]
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC]
+          SortExec: expr=[#12@0 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(u.#9)@0 as #10]
+      AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+        ProjectionExec: expr=[column1@0 as #9]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#6@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#12@0 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13]
+          HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(#2@0, #6@1), (#3@1, #11@0)], projection=[#2@0, #3@1]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                StageInputExec: input=0, partitioning=Hash([#6@0], 4)
+      StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=2, mode=Merge)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#12@0 as k, #13@1 as v]
+      SortPreservingMergeExec: [#12@0 ASC]
+        StageInputExec: input=0, partitioning=Hash([#12@0], 4)
+---
+name: "test_correlated_scalar_subquery_with_nested_scalar_in_projection"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, () AS shifted AS #14
+        Subquery:
+          Projection: max(inner_t.#7) + () AS #11
+            Subquery:
+              Projection: min(u.#9) AS #10
+                Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                  SubqueryAlias: u
+                    Projection: #8 AS #9
+                      Projection: column1 AS #8
+                        Values: (Int32(1))
+            Aggregate: groupBy=[[]], aggr=[[max(inner_t.#7)]]
+              Filter: inner_t.#6 = outer_ref(outer_t.#2)
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+        SubqueryAlias: outer_t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14
+        Left Join:  Filter: __scalar_sq_1.#6 = outer_t.#2
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: #8 AS #9
+                        Projection: column1 AS #8
+                          Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: #8 AS #9
+                        Projection: column1 AS #8
+                          Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6, Boolean(true) AS __always_true]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: #4 AS #6, #5 AS #7
+                    Projection: column1 AS #4, column2 AS #5
+                      Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6, __always_true
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: #8 AS #9
+                        Projection: column1 AS #8
+                          Values: (Int32(1))
+              Projection: inner_t.#6, Boolean(true) AS __always_true, max(inner_t.#7)
+                Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                  SubqueryAlias: inner_t
+                    Projection: #4 AS #6, #5 AS #7
+                      Projection: column1 AS #4, column2 AS #5
+                        Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: column1 AS #9
+                        Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #12 ASC NULLS FIRST, #13 ASC NULLS FIRST
+      Projection: outer_t.#2 AS #12, outer_t.#3 AS #13, __scalar_sq_1.#11 AS shifted AS #14
+        Left Join: outer_t.#2 = __scalar_sq_1.#6
+          SubqueryAlias: outer_t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(4)), (Int32(2), Int32(1)), (Int32(2), Int32(3))
+          SubqueryAlias: __scalar_sq_1
+            Projection: max(inner_t.#7) + () AS #11, inner_t.#6
+              Subquery:
+                Projection: min(u.#9) AS #10
+                  Aggregate: groupBy=[[]], aggr=[[min(u.#9)]]
+                    SubqueryAlias: u
+                      Projection: column1 AS #9
+                        Values: (Int32(1))
+              Aggregate: groupBy=[[inner_t.#6]], aggr=[[max(inner_t.#7)]]
+                SubqueryAlias: inner_t
+                  Projection: column1 AS #6, column2 AS #7
+                    Values: (Int32(1), Int32(4)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(1))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+          HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(u.#9)@0 as #10]
+        AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+      SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:)]]
+          HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], statistics=[Rows=Inexact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0)),(Col[2]:),(Col[3]:)]]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                    DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(u.#9)@0 as #10], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #9], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#12:Int32;N, #13:Int32;N, #14:Int32;N]
+      SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false], schema=[#12:Int32;N, #13:Int32;N, #14:Int32;N]
+        ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14], schema=[#12:Int32;N, #13:Int32;N, #14:Int32;N]
+          HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)], schema=[#2:Int32;N, #3:Int32;N, #11:Int32;N, #6:Int32;N]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+            ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6], schema=[#11:Int32;N, #6:Int32;N]
+              AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7):Int32;N]
+                AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)], schema=[#6:Int32;N, max(inner_t.#7)[value]:Int32;N]
+                  ProjectionExec: expr=[column1@0 as #6, column2@1 as #7], schema=[#6:Int32;N, #7:Int32;N]
+                    DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[min(u.#9)@0 as #10], schema=[#10:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)], schema=[min(u.#9):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)], schema=[min(u.#9)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #9], schema=[#9:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+            HashJoinExec: mode=Auto, join_type=Left, on=[(#2@0, #6@1)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+              ProjectionExec: expr=[column1@0 as #9]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+              ProjectionExec: expr=[column1@0 as #9]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false]
+          CoalescePartitionsExec
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+              HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#9)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#9)]
+              ProjectionExec: expr=[column1@0 as #9]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[false]
+          CoalescePartitionsExec
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+              HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC]
+          SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+              HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC]
+          SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+              HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC]
+        SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+            HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                  RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                        DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(u.#9)@0 as #10]
+        AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+          ProjectionExec: expr=[column1@0 as #9]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC]
+          SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+              HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC]
+          SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true]
+            ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+              HashJoinExec: mode=CollectLeft, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+                  AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                    RepartitionExec: partitioning=Hash([#6@0], 4), input_partitions=1
+                      AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+                        ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+                          DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#9)@0 as #10]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+            ProjectionExec: expr=[column1@0 as #9]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(u.#9)@0 as #10]
+      AggregateExec: mode=Single, gby=[], aggr=[min(u.#9)]
+        ProjectionExec: expr=[column1@0 as #9]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+      DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 2 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#6@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+      ProjectionExec: expr=[column1@0 as #6, column2@1 as #7]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=2, mode=Shuffle), StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[#6@1], channels=4)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[max(inner_t.#7)@1 + scalar_subquery() as #11, #6@0 as #6]
+        AggregateExec: mode=FinalPartitioned, gby=[#6@0 as #6], aggr=[max(inner_t.#7)]
+          StageInputExec: input=0, partitioning=Hash([#6@0], 4)
+      StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+  
+    === stage 4 ===
+    inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=3, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    SortExec: expr=[#12@0 ASC, #13@1 ASC], preserve_partitioning=[true]
+      ProjectionExec: expr=[#2@0 as #12, #3@1 as #13, #11@2 as #14]
+        HashJoinExec: mode=Partitioned, join_type=Left, on=[(#2@0, #6@1)], projection=[#2@0, #3@1, #11@2]
+          StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+          StageInputExec: input=1, partitioning=Hash([#6@1], 4)
+  
+    === stage 5 ===
+    inputs=[StageInput(stage=4, mode=Merge)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#12@0 as k, #13@1 as v, #14@2 as shifted]
+      SortPreservingMergeExec: [#12@0 ASC, #13@1 ASC]
+        StageInputExec: input=0, partitioning=Hash([#12@0], 4)
+---
+name: "test_empty_scalar_subquery_result"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6
+        Subquery:
+          Projection: CAST(s.#3 AS Int64) AS #4
+            Filter: s.#3 > Int32(100)
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(10))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6
+        Subquery:
+          Projection: CAST(s.#3 AS Int64) AS #4
+            SubqueryAlias: s
+              Projection: #2 AS #3
+                Projection: column1 AS #2
+                  Filter: column1 > Int32(100)
+                    Values: (Int32(10))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6
+        Subquery:
+          Projection: CAST(s.#3 AS Int64) AS #4
+            SubqueryAlias: s
+              Projection: column1 AS #3
+                Filter: column1 > Int32(100)
+                  Values: (Int32(10))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS shifted AS #6
+        Subquery:
+          Projection: CAST(s.#3 AS Int64) AS #4
+            SubqueryAlias: s
+              Projection: column1 AS #3
+                Filter: column1 > Int32(100)
+                  Values: (Int32(10))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6]
+          ProjectionExec: expr=[column1@0 as #1]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+        ProjectionExec: expr=[column1@0 as #3]
+          FilterExec: column1@0 > 100
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+        ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+          ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Min=Inexact(Int64(101)) Null=Inexact(0))]]
+        ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Min=Inexact(Int32(101)) Null=Inexact(0))]]
+          FilterExec: column1@0 > 100, statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Min=Inexact(Int32(101)) Null=Inexact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#5:Int32;N, #6:Int64;N]
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], schema=[#5:Int32;N, #6:Int64;N]
+        ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], schema=[#5:Int32;N, #6:Int64;N]
+          ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4], schema=[#4:Int64;N]
+        ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+          FilterExec: column1@0 > 100, schema=[column1:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+          ProjectionExec: expr=[column1@0 as #3]
+            FilterExec: column1@0 > 100
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+            ProjectionExec: expr=[column1@0 as #3]
+              FilterExec: column1@0 > 100
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+            ProjectionExec: expr=[column1@0 as #3]
+              FilterExec: column1@0 > 100
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+        RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+          ProjectionExec: expr=[column1@0 as #3]
+            FilterExec: column1@0 > 100
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#5@0 as v, #6@1 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+            ProjectionExec: expr=[column1@0 as #3]
+              FilterExec: column1@0 > 100
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#5@0 as v, #6@1 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+            ProjectionExec: expr=[column1@0 as #3]
+              FilterExec: column1@0 > 100
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    ProjectionExec: expr=[column1@0 as #3]
+      FilterExec: column1@0 > 100
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[CAST(#3@0 AS Int64) as #4]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#5@0 as v, #6@1 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_multiple_scalar_subqueries_in_one_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: sum(t.#1 + #4 + #7) AS total AS #8
+      Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + () + ())]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(10)), (Int32(20))
+        Subquery:
+          Projection: max(u.#6) AS #7
+            Aggregate: groupBy=[[]], aggr=[[max(CAST(u.#6 AS Int64))]]
+              SubqueryAlias: u
+                Projection: #5 AS #6
+                  Projection: column1 AS #5
+                    Values: (Int32(1)), (Int32(2))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: sum(t.#1 + #4 + #7) AS total AS #8
+      Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + () + ())]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        Subquery:
+          Projection: max(u.#6) AS #7
+            Aggregate: groupBy=[[]], aggr=[[max(CAST(u.#6 AS Int64))]]
+              SubqueryAlias: u
+                Projection: column1 AS #6
+                  Values: (Int32(1)), (Int32(2))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: sum(t.#1 + #4 + #7) AS total AS #8
+      Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + () + ())]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        Subquery:
+          Projection: max(u.#6) AS #7
+            Aggregate: groupBy=[[]], aggr=[[max(CAST(u.#6 AS Int64))]]
+              SubqueryAlias: u
+                Projection: column1 AS #6
+                  Values: (Int32(1)), (Int32(2))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=2
+      ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+        AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+          AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[max(u.#6)@0 as #7]
+        AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)]
+          AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)]
+            ProjectionExec: expr=[column1@0 as #6]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=2, statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+      ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[max(u.#6)@0 as #7], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #6], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=2, schema=[#8:Int64;N]
+      ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8], schema=[#8:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)], schema=[sum(t.#1 + #4 + #7):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)], schema=[sum(t.#1 + #4 + #7)[sum]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[max(u.#6)@0 as #7], schema=[#7:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)], schema=[max(u.#6):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)], schema=[max(u.#6)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #6], schema=[#6:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=2
+        ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+          AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+            AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(u.#6)@0 as #7]
+          AggregateExec: mode=Final, gby=[], aggr=[max(u.#6)]
+            AggregateExec: mode=Partial, gby=[], aggr=[max(u.#6)]
+              ProjectionExec: expr=[column1@0 as #6]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=2
+        ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(u.#6)@0 as #7]
+          AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)]
+            ProjectionExec: expr=[column1@0 as #6]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=2
+      ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+        AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+          ProjectionExec: expr=[column1@0 as #1]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[max(u.#6)@0 as #7]
+        AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)]
+          ProjectionExec: expr=[column1@0 as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#8@0 as total]
+      ScalarSubqueryExec: subqueries=2
+        ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(u.#6)@0 as #7]
+          AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)]
+            ProjectionExec: expr=[column1@0 as #6]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#8@0 as total]
+      ScalarSubqueryExec: subqueries=2
+        ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(u.#6)@0 as #7]
+          AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)]
+            ProjectionExec: expr=[column1@0 as #6]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[max(u.#6)@0 as #7]
+      AggregateExec: mode=Single, gby=[], aggr=[max(u.#6)]
+        ProjectionExec: expr=[column1@0 as #6]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=0, mode=Broadcast), StageInput(stage=1, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#8@0 as total]
+      ScalarSubqueryExec: subqueries=2
+        ProjectionExec: expr=[sum(t.#1 + #4 + #7)@0 as #8]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4 + #7)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+        StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+---
+name: "test_nested_scalar_subquery"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #8 ASC NULLS FIRST
+      Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9
+        Subquery:
+          Projection: min(s.#3) + () AS #7
+            Subquery:
+              Projection: min(u.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(u.#5)]]
+                  SubqueryAlias: u
+                    Projection: #4 AS #5
+                      Projection: column1 AS #4
+                        Values: (Int32(1))
+            Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Sort: #8 ASC NULLS FIRST
+      Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9
+        Subquery:
+          Projection: CAST(min(s.#3) + () AS #7 AS Int64)
+            Subquery:
+              Projection: min(u.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(u.#5)]]
+                  SubqueryAlias: u
+                    Projection: #4 AS #5
+                      Projection: column1 AS #4
+                        Values: (Int32(1))
+            Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2))
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #8 ASC NULLS FIRST
+      Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9
+        Subquery:
+          Projection: CAST(min(s.#3) + () AS #7 AS Int64)
+            Subquery:
+              Projection: min(u.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(u.#5)]]
+                  SubqueryAlias: u
+                    Projection: column1 AS #5
+                      Values: (Int32(1))
+            Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #8 ASC NULLS FIRST
+      Projection: t.#1 AS #8, CAST(t.#1 AS Int64) + () AS shifted AS #9
+        Subquery:
+          Projection: CAST(min(s.#3) + () AS #7 AS Int64)
+            Subquery:
+              Projection: min(u.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(u.#5)]]
+                  SubqueryAlias: u
+                    Projection: column1 AS #5
+                      Values: (Int32(1))
+            Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9]
+          ProjectionExec: expr=[column1@0 as #1]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+      SortExec: expr=[#8@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+        ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+          ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+              ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+        ProjectionExec: expr=[min(u.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+              ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#8:Int32;N, #9:Int64;N]
+      SortExec: expr=[#8@0 ASC], preserve_partitioning=[false], schema=[#8:Int32;N, #9:Int64;N]
+        ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9], schema=[#8:Int32;N, #9:Int64;N]
+          ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ScalarSubqueryExec: subqueries=1, schema=[#7:Int64;N]
+        ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7], schema=[#7:Int64;N]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int32;N]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int32;N]
+              ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+        ProjectionExec: expr=[min(u.#5)@0 as #6], schema=[#6:Int32;N]
+          AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)], schema=[min(u.#5):Int32;N]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)], schema=[min(u.#5)[value]:Int32;N]
+              ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ScalarSubqueryExec: subqueries=1
+          ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+            AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+              AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[min(u.#5)@0 as #6]
+            AggregateExec: mode=Final, gby=[], aggr=[min(u.#5)]
+              AggregateExec: mode=Partial, gby=[], aggr=[min(u.#5)]
+                ProjectionExec: expr=[column1@0 as #5]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #8, CAST(#1@0 AS Int64) + scalar_subquery() as #9]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ScalarSubqueryExec: subqueries=1
+          ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+            AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[min(u.#5)@0 as #6]
+            AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ScalarSubqueryExec: subqueries=1
+          ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+            AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[min(u.#5)@0 as #6]
+            AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(u.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#8@0 as v, #9@1 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ScalarSubqueryExec: subqueries=1
+          ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+            AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[min(u.#5)@0 as #6]
+            AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#8@0 as v, #9@1 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ScalarSubqueryExec: subqueries=1
+          ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+            AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+          ProjectionExec: expr=[min(u.#5)@0 as #6]
+            AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(u.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(u.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[CAST(min(s.#3)@0 + scalar_subquery() AS Int64) as #7]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#8@0 as v, #9@1 as shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#8@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #8, CAST(column1@0 AS Int64) + scalar_subquery() as #9]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_null_scalar_subquery_result"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (CAST(NULL AS Int32))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(NULL) AS NULL)
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(NULL) AS NULL)
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #5 ASC NULLS FIRST
+      Projection: t.#1 AS #5, CAST(t.#1 AS Int64) + () AS maybe_shifted AS #6
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(NULL) AS NULL)
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6]
+          ProjectionExec: expr=[column1@0 as #1]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+        ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+          ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(1))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(1))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#5:Int32;N, #6:Int64;N]
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false], schema=[#5:Int32;N, #6:Int64;N]
+        ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6], schema=[#5:Int32;N, #6:Int64;N]
+          ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #5, CAST(#1@0 AS Int64) + scalar_subquery() as #6]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#5@0 as v, #6@1 as maybe_shifted]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#5@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #5, CAST(column1@0 AS Int64) + scalar_subquery() as #6]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_aggregate_filter"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: t.#2 AS #7, count(Int64(1)) FILTER (WHERE t.#3 > ()) AS n AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[count(Int64(1)) FILTER (WHERE t.#3 > ())]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(1))
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: t.#2 AS #7, count(Int64(1)) FILTER (WHERE t.#3 > ()) AS n AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[count(Int64(1)) FILTER (WHERE t.#3 > ())]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: column1 AS #5
+                  Values: (Int32(1))
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: t.#2 AS #7, count(Int64(1)) FILTER (WHERE t.#3 > ()) AS n AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[count(Int64(1)) FILTER (WHERE t.#3 > ())]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: column1 AS #5
+                  Values: (Int32(1))
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64]
+      ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8], schema=[#7:Int32;N, #8:Int64]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], schema=[#2:Int32;N, count(Int64(1)) FILTER (WHERE t.#3 > ()):Int64]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())], schema=[#2:Int32;N, count(Int64(1)) FILTER (WHERE t.#3 > ())[count]:Int64]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+          RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as k, #8@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as k, #8@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+        ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as k, #8@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, count(Int64(1)) FILTER (WHERE t.#3 > ())@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[count(Int64(1)) FILTER (WHERE t.#3 > ())]
+            StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+        StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_aggregate_order_by"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: CASE WHEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) IS NOT NULL THEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) WHEN Boolean(true) THEN NULL END AS joined AS #7
+      Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(1))
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Projection: CASE WHEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) IS NOT NULL THEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) WHEN Boolean(true) THEN CAST(NULL AS Utf8) END AS joined AS #7
+      Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(1))
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3))
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Projection: CASE WHEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) IS NOT NULL THEN array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) ELSE Utf8(NULL) END AS joined AS #7
+      Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(1))
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3))
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    Projection: CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Utf8(NULL) END AS joined AS #7
+      Projection: array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) AS __common_expr_1, array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]
+        Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]]
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3))
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Utf8(NULL) END AS joined AS #7
+      Projection: array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) AS __common_expr_1
+        Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]]
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+                SubqueryAlias: s
+                  Projection: column1 AS #5
+                    Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Utf8(NULL) END AS joined AS #7
+      Projection: array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST], Utf8(",")) AS __common_expr_1
+        Aggregate: groupBy=[[]], aggr=[[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] AS array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]]
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+                SubqueryAlias: s
+                  Projection: column1 AS #5
+                    Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Utf8("a"), Int32(2)), (Utf8("b"), Int32(1)), (Utf8("c"), Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+        ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+          AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+            AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+        ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+            AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Utf8;N]
+      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7], schema=[#7:Utf8;N]
+        ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1], schema=[__common_expr_1:Utf8;N]
+          AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], schema=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]:List(Utf8);N]
+            AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]], schema=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST][array_agg]:List(Utf8);N, array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST][array_agg_orderings]:List(Struct("CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END": Int32))]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Utf8;N, #3:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Utf8;N, column2:Int32;N]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+          ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+            AggregateExec: mode=Final, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              AggregateExec: mode=Partial, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+          ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+            AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+          ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+            AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+        ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+          AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+            SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as joined]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+          ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+            AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as joined]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+          ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+            AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as joined]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 END as #7]
+          ProjectionExec: expr=[array_to_string(array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]@0, ,) as __common_expr_1]
+            AggregateExec: mode=Single, gby=[], aggr=[array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 ELSE negative(t.#3) END ASC NULLS FIRST] as array_agg(t.#2) IGNORE NULLS ORDER BY [CASE WHEN () = Int32(1) THEN t.#3 WHEN Boolean(true) THEN negative(t.#3) END ASC NULLS FIRST]]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #3@1 ELSE negative(#3@1) END ASC], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_filter_before_aggregate"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(t.#3)]]
+        Filter: t.#3 > ()
+          Subquery:
+            Projection: max(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[max(s.#5)]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(1)), (Int32(2))
+          SubqueryAlias: t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        Filter: t.#3 > ()
+          Subquery:
+            Projection: max(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[max(s.#5)]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(1)), (Int32(2))
+          SubqueryAlias: t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1))
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Filter: column2 > ()
+                Subquery:
+                  Projection: max(s.#5) AS #6
+                    Aggregate: groupBy=[[]], aggr=[[max(s.#5)]]
+                      SubqueryAlias: s
+                        Projection: #4 AS #5
+                          Projection: column1 AS #4
+                            Values: (Int32(1)), (Int32(2))
+                Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1))
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Filter: column2 > ()
+              Subquery:
+                Projection: max(s.#5) AS #6
+                  Aggregate: groupBy=[[]], aggr=[[max(s.#5)]]
+                    SubqueryAlias: s
+                      Projection: column1 AS #5
+                        Values: (Int32(1)), (Int32(2))
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Filter: column2 > ()
+              Subquery:
+                Projection: max(s.#5) AS #6
+                  Aggregate: groupBy=[[]], aggr=[[max(s.#5)]]
+                    SubqueryAlias: s
+                      Projection: column1 AS #5
+                        Values: (Int32(1)), (Int32(2))
+              Values: (Int32(1), Int32(2)), (Int32(1), Int32(3)), (Int32(2), Int32(10)), (Int32(3), Int32(1))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              FilterExec: column2@1 > scalar_subquery()
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[max(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]]
+              FilterExec: column2@1 > scalar_subquery(), statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[max(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64;N]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], schema=[#7:Int32;N, #8:Int64;N]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3):Int64;N]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3)[sum]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+              FilterExec: column2@1 > scalar_subquery(), schema=[column1:Int32;N, column2:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[max(s.#5)@0 as #6], schema=[#6:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)], schema=[max(s.#5):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)], schema=[max(s.#5)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                FilterExec: column2@1 > scalar_subquery()
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    FilterExec: column2@1 > scalar_subquery()
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[max(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[max(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    FilterExec: column2@1 > scalar_subquery()
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  FilterExec: column2@1 > scalar_subquery()
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[max(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    FilterExec: column2@1 > scalar_subquery()
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=4
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    FilterExec: column2@1 > scalar_subquery()
+                      DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[max(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[max(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[max(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=4)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+        FilterExec: column2@1 > scalar_subquery()
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+      StageInputExec: input=0, partitioning=RoundRobinBatch(4)
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=2, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+          StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+---
+name: "test_scalar_subquery_in_group_by_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: g AS #5, count(Int64(1)) AS n AS #6
+      Aggregate: groupBy=[[CAST(t.#1 AS Int64) + () AS g]], aggr=[[count(Int64(1))]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2)), (Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: g AS #5, count(Int64(1)) AS n AS #6
+      Aggregate: groupBy=[[CAST(t.#1 AS Int64) + () AS g]], aggr=[[count(Int64(1))]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(1))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: g AS #5, count(Int64(1)) AS n AS #6
+      Aggregate: groupBy=[[CAST(t.#1 AS Int64) + () AS g]], aggr=[[count(Int64(1))]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(1))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+        AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+          AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#5:Int64;N, #6:Int64]
+      ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6], schema=[#5:Int64;N, #6:Int64]
+        AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))], schema=[g:Int64;N, count(Int64(1)):Int64]
+          AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))], schema=[g:Int64;N, count(Int64(1))[count]:Int64]
+            ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+          AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+            AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+          AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+            RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+          AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+            RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+        AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+          RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#5@0 as g, #6@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+          AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+            RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#5@0 as g, #6@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+          AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+            RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[g@0], channels=4)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      AggregateExec: mode=Partial, gby=[CAST(#1@0 AS Int64) + scalar_subquery() as g], aggr=[count(Int64(1))]
+        ProjectionExec: expr=[column1@0 as #1]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#5@0 as g, #6@1 as n]
+      ProjectionExec: expr=[g@0 as #5, count(Int64(1))@1 as #6]
+        AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[count(Int64(1))]
+          StageInputExec: input=0, partitioning=Hash([g@0], 4)
+---
+name: "test_scalar_subquery_in_grouped_aggregate_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: t.#2 AS #7, sum(t.#3 + #6) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64) + ())]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: #0 AS #2, #1 AS #3
+            Projection: column1 AS #0, column2 AS #1
+              Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: t.#2 AS #7, sum(t.#3 + #6) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64) + ())]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #5
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: t.#2 AS #7, sum(t.#3 + #6) AS total AS #8
+      Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64) + ())]]
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #5
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #2, column2 AS #3
+            Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64;N]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8], schema=[#7:Int32;N, #8:Int64;N]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], schema=[#2:Int32;N, sum(t.#3 + #6):Int64;N]
+          AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)], schema=[#2:Int32;N, sum(t.#3 + #6)[sum]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+        AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+          RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+        ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3 + #6)@1 as #8]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3 + #6)]
+            StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+        StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_having_filter"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Filter: sum(t.#3) > ()
+        Subquery:
+          Projection: min(s.#5) AS #6
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(3))
+        Aggregate: groupBy=[[t.#2]], aggr=[[sum(t.#3)]]
+          SubqueryAlias: t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Filter: sum(t.#3) > ()
+        Subquery:
+          Projection: CAST(min(s.#5) AS #6 AS Int64)
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: #4 AS #5
+                  Projection: column1 AS #4
+                    Values: (Int32(3))
+        Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+          SubqueryAlias: t
+            Projection: #0 AS #2, #1 AS #3
+              Projection: column1 AS #0, column2 AS #1
+                Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4))
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Filter: sum(t.#3) > ()
+        Subquery:
+          Projection: CAST(min(s.#5) AS #6 AS Int64)
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: column1 AS #5
+                  Values: (Int32(3))
+        Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+          SubqueryAlias: t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: t.#2 AS #7, sum(t.#3) AS total AS #8
+      Filter: sum(t.#3) > ()
+        Subquery:
+          Projection: CAST(min(s.#5) AS #6 AS Int64)
+            Aggregate: groupBy=[[]], aggr=[[min(s.#5)]]
+              SubqueryAlias: s
+                Projection: column1 AS #5
+                  Values: (Int32(3))
+        Aggregate: groupBy=[[t.#2]], aggr=[[sum(CAST(t.#3 AS Int64))]]
+          SubqueryAlias: t
+            Projection: column1 AS #2, column2 AS #3
+              Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3)), (Int32(2), Int32(4))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+        FilterExec: sum(t.#3)@1 > scalar_subquery()
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        FilterExec: sum(t.#3)@1 > scalar_subquery(), statistics=[Rows=Inexact(1), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], statistics=[Rows=Inexact(4), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int64;N]
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8], schema=[#7:Int32;N, #8:Int64;N]
+        FilterExec: sum(t.#3)@1 > scalar_subquery(), schema=[#2:Int32;N, sum(t.#3):Int64;N]
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3):Int64;N]
+            AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)], schema=[#2:Int32;N, sum(t.#3)[sum]:Int64;N]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6], schema=[#6:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          FilterExec: sum(t.#3)@1 > scalar_subquery()
+            AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          FilterExec: sum(t.#3)@1 > scalar_subquery()
+            AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+                AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          FilterExec: sum(t.#3)@1 > scalar_subquery()
+            AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+                AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+        FilterExec: sum(t.#3)@1 > scalar_subquery()
+          AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+            RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          FilterExec: sum(t.#3)@1 > scalar_subquery()
+            AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+                AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          FilterExec: sum(t.#3)@1 > scalar_subquery()
+            AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              RepartitionExec: partitioning=Hash([#2@0], 4), input_partitions=1
+                AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[CAST(min(s.#5)@0 AS Int64) as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#2@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+      ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle), StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as k, #8@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#2@0 as #7, sum(t.#3)@1 as #8]
+          FilterExec: sum(t.#3)@1 > scalar_subquery()
+            AggregateExec: mode=FinalPartitioned, gby=[#2@0 as #2], aggr=[sum(t.#3)]
+              StageInputExec: input=0, partitioning=Hash([#2@0], 4)
+        StageInputExec: input=1, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_join_condition"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #7 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Inner Join:  Filter: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: l
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3))
+          SubqueryAlias: r
+            Projection: #2 AS #3
+              Projection: column1 AS #2
+                Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    Sort: #7 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Inner Join: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: l
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3))
+          SubqueryAlias: r
+            Projection: #2 AS #3
+              Projection: column1 AS #2
+                Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #7 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Inner Join: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: column1 AS #5
+                    Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: l
+            Projection: column1 AS #1
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+          SubqueryAlias: r
+            Projection: column1 AS #3
+              Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #7 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Inner Join: CAST(l.#1 AS Int64) + () = CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: column1 AS #5
+                    Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: l
+            Projection: column1 AS #1
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+          SubqueryAlias: r
+            Projection: column1 AS #3
+              Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+          ProjectionExec: expr=[#1@0 as #1, #3@2 as #3]
+            HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)]
+              ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      SortExec: expr=[#7@0 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+          ProjectionExec: expr=[#1@0 as #1, #3@2 as #3], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+            HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], statistics=[Rows=Inexact(3), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:),(Col[2]: Null=Exact(0)),(Col[3]:)]]
+              ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+                ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+                  DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+                ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+                  DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int32;N]
+      SortExec: expr=[#7@0 ASC], preserve_partitioning=[false], schema=[#7:Int32;N, #8:Int32;N]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], schema=[#7:Int32;N, #8:Int32;N]
+          ProjectionExec: expr=[#1@0 as #1, #3@2 as #3], schema=[#1:Int32;N, #3:Int32;N]
+            HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], schema=[#1:Int32;N, l.#1 + #6:Int64;N, #3:Int32;N, CAST(r.#3 AS Int64):Int64;N]
+              ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6], schema=[#1:Int32;N, l.#1 + #6:Int64;N]
+                ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+                  DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+              ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)], schema=[#3:Int32;N, CAST(r.#3 AS Int64):Int64;N]
+                ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+                  DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            ProjectionExec: expr=[#1@0 as #1, #3@2 as #3]
+              HashJoinExec: mode=Auto, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)]
+                ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                  ProjectionExec: expr=[column1@0 as #1]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)]
+                  ProjectionExec: expr=[column1@0 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            ProjectionExec: expr=[#1@0 as #1, #3@2 as #3]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)]
+                ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                  ProjectionExec: expr=[column1@0 as #1]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)]
+                  ProjectionExec: expr=[column1@0 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            ProjectionExec: expr=[#1@0 as #1, #3@2 as #3]
+              HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)]
+                ProjectionExec: expr=[#1@0 as #1, CAST(#1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                  ProjectionExec: expr=[column1@0 as #1]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[#3@0 as #3, CAST(#3@0 AS Int64) as CAST(r.#3 AS Int64)]
+                  ProjectionExec: expr=[column1@0 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2]
+              ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+          HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2]
+            ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+            ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as v, #8@1 as w]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2]
+              ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as v, #8@1 as w]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2]
+              ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as v, #8@1 as w]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l.#1 + #6@1, CAST(r.#3 AS Int64)@1)], projection=[#1@0, #3@2]
+              ProjectionExec: expr=[column1@0 as #1, CAST(column1@0 AS Int64) + scalar_subquery() as l.#1 + #6]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[column1@0 as #3, CAST(column1@0 AS Int64) as CAST(r.#3 AS Int64)]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_nonequi_join_condition"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Inner Join:  Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: l
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3))
+          SubqueryAlias: r
+            Projection: #2 AS #3
+              Projection: column1 AS #2
+                Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: #4 AS #5
+                    Projection: column1 AS #4
+                      Values: (Int32(10)), (Int32(20))
+          Cross Join:
+            SubqueryAlias: l
+              Projection: #0 AS #1
+                Projection: column1 AS #0
+                  Values: (Int32(1)), (Int32(2)), (Int32(3))
+            SubqueryAlias: r
+              Projection: #2 AS #3
+                Projection: column1 AS #2
+                  Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: column1 AS #5
+                    Values: (Int32(10)), (Int32(20))
+          Cross Join:
+            SubqueryAlias: l
+              Projection: column1 AS #1
+                Values: (Int32(1)), (Int32(2)), (Int32(3))
+            SubqueryAlias: r
+              Projection: column1 AS #3
+                Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #7 ASC NULLS FIRST, #8 ASC NULLS FIRST
+      Projection: l.#1 AS #7, r.#3 AS #8
+        Filter: CAST(l.#1 AS Int64) + () < CAST(r.#3 AS Int64)
+          Subquery:
+            Projection: min(s.#5) AS #6
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: column1 AS #5
+                    Values: (Int32(10)), (Int32(20))
+          Cross Join:
+            SubqueryAlias: l
+              Projection: column1 AS #1
+                Values: (Int32(1)), (Int32(2)), (Int32(3))
+            SubqueryAlias: r
+              Projection: column1 AS #3
+                Values: (Int32(11)), (Int32(12)), (Int32(13))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+          FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+            CrossJoinExec
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]]
+      SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]]
+          FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64), statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]: Null=Inexact(0)),(Col[1]: Null=Inexact(0))]]
+            CrossJoinExec, statistics=[Rows=Exact(9), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+              ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#7:Int32;N, #8:Int32;N]
+      SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false], schema=[#7:Int32;N, #8:Int32;N]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8], schema=[#7:Int32;N, #8:Int32;N]
+          FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64), schema=[#1:Int32;N, #3:Int32;N]
+            CrossJoinExec, schema=[#1:Int32;N, #3:Int32;N]
+              ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+              ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+              CrossJoinExec
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+              CrossJoinExec
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+          FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+            CrossJoinExec
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#7@0 as v, #8@1 as w]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+              CrossJoinExec
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#7@0 as v, #8@1 as w]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+              CrossJoinExec
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#7@0 as v, #8@1 as w]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@0 ASC, #8@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #7, #3@1 as #8]
+            FilterExec: CAST(#1@0 AS Int64) + scalar_subquery() < CAST(#3@1 AS Int64)
+              CrossJoinExec
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+                ProjectionExec: expr=[column1@0 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_projection_before_aggregate"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: p.#7 AS #9, sum(p.#8) AS total AS #10
+      Aggregate: groupBy=[[p.#7]], aggr=[[sum(p.#8)]]
+        SubqueryAlias: p
+          Projection: t.#2 AS #7, CAST(t.#3 AS Int64) + () AS shifted AS #8
+            Subquery:
+              Projection: min(s.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                  SubqueryAlias: s
+                    Projection: #4 AS #5
+                      Projection: column1 AS #4
+                        Values: (Int32(10)), (Int32(20))
+            SubqueryAlias: t
+              Projection: #0 AS #2, #1 AS #3
+                Projection: column1 AS #0, column2 AS #1
+                  Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: p.#7 AS #9, sum(p.#8) AS total AS #10
+      Aggregate: groupBy=[[p.#7]], aggr=[[sum(p.#8)]]
+        SubqueryAlias: p
+          Projection: t.#2 AS #7, CAST(t.#3 AS Int64) + () AS shifted AS #8
+            Subquery:
+              Projection: min(s.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                  SubqueryAlias: s
+                    Projection: column1 AS #5
+                      Values: (Int32(10)), (Int32(20))
+            SubqueryAlias: t
+              Projection: column1 AS #2, column2 AS #3
+                Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: p.#7 AS #9, sum(p.#8) AS total AS #10
+      Aggregate: groupBy=[[p.#7]], aggr=[[sum(p.#8)]]
+        SubqueryAlias: p
+          Projection: t.#2 AS #7, CAST(t.#3 AS Int64) + () AS shifted AS #8
+            Subquery:
+              Projection: min(s.#5) AS #6
+                Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#5 AS Int64))]]
+                  SubqueryAlias: s
+                    Projection: column1 AS #5
+                      Values: (Int32(10)), (Int32(20))
+            SubqueryAlias: t
+              Projection: column1 AS #2, column2 AS #3
+                Values: (Int32(1), Int32(1)), (Int32(1), Int32(2)), (Int32(2), Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+        AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+          AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+      ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+        AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+          AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:),(Col[1]:)]]
+            ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+                DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #5], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#9:Int32;N, #10:Int64;N]
+      ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10], schema=[#9:Int32;N, #10:Int64;N]
+        AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)], schema=[#7:Int32;N, sum(p.#8):Int64;N]
+          AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)], schema=[#7:Int32;N, sum(p.#8)[sum]:Int64;N]
+            ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8], schema=[#7:Int32;N, #8:Int64;N]
+              ProjectionExec: expr=[column1@0 as #2, column2@1 as #3], schema=[#2:Int32;N, #3:Int32;N]
+                DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N, column2:Int32;N]
+      ProjectionExec: expr=[min(s.#5)@0 as #6], schema=[#6:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)], schema=[min(s.#5):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)], schema=[min(s.#5)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #5], schema=[#5:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+          AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+              ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8]
+                ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+          AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+                ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8]
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#5)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#5)]
+              ProjectionExec: expr=[column1@0 as #5]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+          AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+                ProjectionExec: expr=[#2@0 as #7, CAST(#3@1 AS Int64) + scalar_subquery() as #8]
+                  ProjectionExec: expr=[column1@0 as #2, column2@1 as #3]
+                    DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+          AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+                ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+        AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+          RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1
+            AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+              ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#5)@0 as #6]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+          ProjectionExec: expr=[column1@0 as #5]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#9@0 as k, #10@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+          AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+                ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#9@0 as k, #10@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+          AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+            RepartitionExec: partitioning=Hash([#7@0], 4), input_partitions=1
+              AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+                ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#5)@0 as #6]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+            ProjectionExec: expr=[column1@0 as #5]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#5)@0 as #6]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#5)]
+        ProjectionExec: expr=[column1@0 as #5]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=Hash(keys=[#7@0], channels=4)
+    placement=Worker
+    ScalarSubqueryExec: subqueries=1
+      AggregateExec: mode=Partial, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+        ProjectionExec: expr=[column1@0 as #7, CAST(column2@1 AS Int64) + scalar_subquery() as #8]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+  
+    === stage 2 ===
+    inputs=[StageInput(stage=1, mode=Shuffle)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#9@0 as k, #10@1 as total]
+      ProjectionExec: expr=[#7@0 as #9, sum(p.#8)@1 as #10]
+        AggregateExec: mode=FinalPartitioned, gby=[#7@0 as #7], aggr=[sum(p.#8)]
+          StageInputExec: input=0, partitioning=Hash([#7@0], 4)
+---
+name: "test_scalar_subquery_in_sort_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: CASE WHEN () = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST
+      Subquery:
+        Projection: min(s.#4) AS #5
+          Aggregate: groupBy=[[]], aggr=[[min(s.#4)]]
+            SubqueryAlias: s
+              Projection: #3 AS #4
+                Projection: column1 AS #3
+                  Values: (Int32(2))
+      Projection: t.#1 AS #2
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Sort: CASE WHEN () = Int32(2) THEN #2 ELSE negative(#2) END AS CASE WHEN #5 = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST
+      Subquery:
+        Projection: min(s.#4) AS #5
+          Aggregate: groupBy=[[]], aggr=[[min(s.#4)]]
+            SubqueryAlias: s
+              Projection: #3 AS #4
+                Projection: column1 AS #3
+                  Values: (Int32(2))
+      Projection: t.#1 AS #2
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: CASE WHEN () = Int32(2) THEN #2 ELSE negative(#2) END AS CASE WHEN #5 = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST
+      Subquery:
+        Projection: min(s.#4) AS #5
+          Aggregate: groupBy=[[]], aggr=[[min(s.#4)]]
+            SubqueryAlias: s
+              Projection: column1 AS #4
+                Values: (Int32(2))
+      Projection: t.#1 AS #2
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: CASE WHEN () = Int32(2) THEN #2 ELSE negative(#2) END AS CASE WHEN #5 = Int32(2) THEN #2 WHEN Boolean(true) THEN negative(#2) END DESC NULLS LAST
+      Subquery:
+        Projection: min(s.#4) AS #5
+          Aggregate: groupBy=[[]], aggr=[[min(s.#4)]]
+            SubqueryAlias: s
+              Projection: column1 AS #4
+                Values: (Int32(2))
+      Projection: t.#1 AS #2
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #2]
+          ProjectionExec: expr=[column1@0 as #1]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#4)@0 as #5]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)]
+            ProjectionExec: expr=[column1@0 as #4]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+        ProjectionExec: expr=[#1@0 as #2], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+          ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+            DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#4)@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#2:Int32;N]
+      SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false], schema=[#2:Int32;N]
+        ProjectionExec: expr=[#1@0 as #2], schema=[#2:Int32;N]
+          ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+            DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#4)@0 as #5], schema=[#5:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)], schema=[min(s.#4):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)], schema=[min(s.#4)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #4], schema=[#4:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #2]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#4)@0 as #5]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#4)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#4)]
+              ProjectionExec: expr=[column1@0 as #4]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #2]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)]
+            ProjectionExec: expr=[column1@0 as #4]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #2]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)]
+            ProjectionExec: expr=[column1@0 as #4]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+        ProjectionExec: expr=[column1@0 as #2]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#4)@0 as #5]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)]
+          ProjectionExec: expr=[column1@0 as #4]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#2@0 as v]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #2]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)]
+            ProjectionExec: expr=[column1@0 as #4]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as v]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #2]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)]
+            ProjectionExec: expr=[column1@0 as #4]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#4)@0 as #5]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#4)]
+        ProjectionExec: expr=[column1@0 as #4]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#2@0 as v]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #2@0 ELSE negative(#2@0) END DESC NULLS LAST], preserve_partitioning=[false]
+          ProjectionExec: expr=[column1@0 as #2]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_ungrouped_aggregate_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Projection: sum(t.#1 + #4) AS total AS #5
+      Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + ())]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: #2 AS #3
+                  Projection: column1 AS #2
+                    Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: #0 AS #1
+            Projection: column1 AS #0
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Projection: sum(t.#1 + #4) AS total AS #5
+      Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + ())]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Projection: sum(t.#1 + #4) AS total AS #5
+      Aggregate: groupBy=[[]], aggr=[[sum(CAST(t.#1 AS Int64) + ())]]
+        Subquery:
+          Projection: min(s.#3) AS #4
+            Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+              SubqueryAlias: s
+                Projection: column1 AS #3
+                  Values: (Int32(10)), (Int32(20))
+        SubqueryAlias: t
+          Projection: column1 AS #1
+            Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+        AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)]
+          AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+      ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)], statistics=[Rows=Inexact(3), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#5:Int64;N]
+      ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5], schema=[#5:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)], schema=[sum(t.#1 + #4):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)], schema=[sum(t.#1 + #4)[sum]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+          AggregateExec: mode=Final, gby=[], aggr=[sum(t.#1 + #4)]
+            AggregateExec: mode=Partial, gby=[], aggr=[sum(t.#1 + #4)]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+        AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)]
+          ProjectionExec: expr=[column1@0 as #1]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#5@0 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#5@0 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#5@0 as total]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[sum(t.#1 + #4)@0 as #5]
+          AggregateExec: mode=Single, gby=[], aggr=[sum(t.#1 + #4)]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_window_aggregate_argument"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS total AS #7
+        WindowAggr: windowExpr=[[sum(CAST(t.#1 AS Int64) + ()) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: #2 AS #3
+                    Projection: column1 AS #2
+                      Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: t
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS total AS #7
+        WindowAggr: windowExpr=[[sum(CAST(t.#1 AS Int64) + ()) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: column1 AS #3
+                    Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: t
+            Projection: column1 AS #1
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS total AS #7
+        WindowAggr: windowExpr=[[sum(CAST(t.#1 AS Int64) + ()) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(CAST(s.#3 AS Int64))]]
+                SubqueryAlias: s
+                  Projection: column1 AS #3
+                    Values: (Int32(10)), (Int32(20))
+          SubqueryAlias: t
+            Projection: column1 AS #1
+              Values: (Int32(1)), (Int32(2)), (Int32(3))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+          WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+        ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+          WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], statistics=[Rows=Exact(3), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Inexact(), [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Inexact(2), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(2), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#6:Int32;N, #7:Int64;N]
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], schema=[#6:Int32;N, #7:Int64;N]
+        ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], schema=[#6:Int32;N, #7:Int64;N]
+          WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], schema=[#1:Int32;N, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING:Int64;N]
+            ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int64;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int64;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int64;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+          WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#6@0 as v, #7@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as v, #7@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#6@0 as v, #7@1 as total]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t.#1 + #4) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_window_order_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #7 ASC NULLS FIRST
+      Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7
+        WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: #2 AS #3
+                    Projection: column1 AS #2
+                      Values: (Int32(2))
+          SubqueryAlias: t
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(3)), (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    SAME TEXT AS ABOVE
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Sort: #7 ASC NULLS FIRST
+      Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7
+        WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 ELSE negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: #2 AS #3
+                    Projection: column1 AS #2
+                      Values: (Int32(2))
+          SubqueryAlias: t
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(3)), (Int32(1)), (Int32(2))
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #7 ASC NULLS FIRST
+      Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7
+        WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 ELSE negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: column1 AS #3
+                    Values: (Int32(2))
+          SubqueryAlias: t
+            Projection: column1 AS #1
+              Values: (Int32(3)), (Int32(1)), (Int32(2))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #7 ASC NULLS FIRST
+      Projection: t.#1 AS #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS #5 AS Int32) AS rn AS #7
+        WindowAggr: windowExpr=[[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 ELSE negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: column1 AS #3
+                    Values: (Int32(2))
+          SubqueryAlias: t
+            Projection: column1 AS #1
+              Values: (Int32(3)), (Int32(1)), (Int32(2))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#7@1 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+      SortExec: expr=[#7@1 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted], statistics=[Rows=Exact(3), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(3), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#6:Int32;N, #7:Int32]
+      SortExec: expr=[#7@1 ASC], preserve_partitioning=[false], schema=[#6:Int32;N, #7:Int32]
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7], schema=[#6:Int32;N, #7:Int32]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted], schema=[#1:Int32;N, row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]
+            ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+            BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#7@1 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+            BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+            SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+        BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+          SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#6@0 as v, #7@1 as rn]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+            SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as v, #7@1 as rn]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+            SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#6@0 as v, #7@1 as rn]
+      ScalarSubqueryExec: subqueries=1
+        ProjectionExec: expr=[#1@0 as #6, CAST(row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 AS Int32) as #7]
+          BoundedWindowAggExec: wdw=[row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [CASE WHEN () = Int32(2) THEN t.#1 WHEN Boolean(true) THEN negative(t.#1) END ASC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+            SortExec: expr=[CASE WHEN scalar_subquery() = 2 THEN #1@0 ELSE negative(#1@0) END ASC], preserve_partitioning=[false]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
+---
+name: "test_scalar_subquery_in_window_partition_expression"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7
+        WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: #2 AS #3
+                    Projection: column1 AS #2
+                      Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7
+        WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN CAST(raise_error(Utf8("Remainder by zero")) AS Int32) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: #2 AS #3
+                    Projection: column1 AS #2
+                      Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4))
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7
+        WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % Int32(2) ELSE t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: #2 AS #3
+                    Projection: column1 AS #2
+                      Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: #0 AS #1
+              Projection: column1 AS #0
+                Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4))
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7
+        WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % Int32(2) ELSE t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: column1 AS #3
+                    Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: column1 AS #1
+              Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4))
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #6 ASC NULLS FIRST
+      Projection: t.#1 AS #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS #5 AS n AS #7
+        WindowAggr: windowExpr=[[count(Int64(1)) PARTITION BY [CASE WHEN () = Int32(1) THEN t.#1 % Int32(2) ELSE t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]
+          Subquery:
+            Projection: min(s.#3) AS #4
+              Aggregate: groupBy=[[]], aggr=[[min(s.#3)]]
+                SubqueryAlias: s
+                  Projection: column1 AS #3
+                    Values: (Int32(1))
+          SubqueryAlias: t
+            Projection: column1 AS #1
+              Values: (Int32(1)), (Int32(2)), (Int32(3)), (Int32(4))
+  
+    initial_physical_plan:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+          WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+            ProjectionExec: expr=[column1@0 as #1]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    initial_physical_plan_with_stats:
+    ScalarSubqueryExec: subqueries=1, statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+        ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+          WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], statistics=[Rows=Exact(4), Bytes=Absent, [(Col[0]: Null=Exact(0)),(Col[1]:)]]
+            ProjectionExec: expr=[column1@0 as #1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(4), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]:)]]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(1), Bytes=Absent, [(Col[0]:)]]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], statistics=[Rows=Exact(0), Bytes=Inexact(), [(Col[0]:)]]
+            ProjectionExec: expr=[column1@0 as #3], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+              DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(1), Bytes=Exact(), [(Col[0]: Null=Exact(0))]]
+  
+  
+    initial_physical_plan_with_schema:
+    ScalarSubqueryExec: subqueries=1, schema=[#6:Int32;N, #7:Int64]
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false], schema=[#6:Int32;N, #7:Int64]
+        ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7], schema=[#6:Int32;N, #7:Int64]
+          WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }], schema=[#1:Int32;N, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING:Int64]
+            ProjectionExec: expr=[column1@0 as #1], schema=[#1:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+      ProjectionExec: expr=[min(s.#3)@0 as #4], schema=[#4:Int32;N]
+        AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)], schema=[min(s.#3):Int32;N]
+          AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)], schema=[min(s.#3)[value]:Int32;N]
+            ProjectionExec: expr=[column1@0 as #3], schema=[#3:Int32;N]
+              DataSourceExec: partitions=1, partition_sizes=[1], schema=[column1:Int32;N]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Final, gby=[], aggr=[min(s.#3)]
+            AggregateExec: mode=Partial, gby=[], aggr=[min(s.#3)]
+              ProjectionExec: expr=[column1@0 as #3]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after CombinePartialFinalAggregate:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[], dist_by=Unspecified
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after OutputRequirements:
+    ScalarSubqueryExec: subqueries=1
+      SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+          WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+            SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false]
+              ProjectionExec: expr=[column1@0 as #1]
+                DataSourceExec: partitions=1, partition_sizes=[1]
+      ProjectionExec: expr=[min(s.#3)@0 as #4]
+        AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+          ProjectionExec: expr=[column1@0 as #3]
+            DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#6@0 as v, #7@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as v, #7@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        ProjectionExec: expr=[min(s.#3)@0 as #4]
+          AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+            ProjectionExec: expr=[column1@0 as #3]
+              DataSourceExec: partitions=1, partition_sizes=[1]
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[min(s.#3)@0 as #4]
+      AggregateExec: mode=Single, gby=[], aggr=[min(s.#3)]
+        ProjectionExec: expr=[column1@0 as #3]
+          DataSourceExec: partitions=1, partition_sizes=[1]
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Broadcast)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#6@0 as v, #7@1 as n]
+      ScalarSubqueryExec: subqueries=1
+        SortExec: expr=[#6@0 ASC], preserve_partitioning=[false]
+          ProjectionExec: expr=[#1@0 as #6, count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@1 as #7]
+            WindowAggExec: wdw=[count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "count(Int64(1)) PARTITION BY [CASE WHEN #4 = Int32(1) THEN t.#1 % CASE WHEN Int32(2) = Int32(0) THEN raise_error(Utf8(/"Remainder by zero/")) ELSE Int32(2) END WHEN Boolean(true) THEN t.#1 END] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64 }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]
+              SortExec: expr=[CASE WHEN scalar_subquery() = 1 THEN #1@0 % 2 ELSE #1@0 END ASC NULLS LAST], preserve_partitioning=[false]
+                ProjectionExec: expr=[column1@0 as #1]
+                  DataSourceExec: partitions=1, partition_sizes=[1]
+        StageInputExec: input=0, partitioning=UnknownPartitioning(1)
diff --git a/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml b/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml
new file mode 100644
index 000000000..db354c5f3
--- /dev/null
+++ b/python/pysail/tests/spark/execution/__snapshots__/test_shuffle_storage.plan.yaml
@@ -0,0 +1,658 @@
+# serializer version: 1
+---
+name: "test_query_execution_with_storage_shuffle"
+data:
+  |-
+    == Codegen ==
+    Whole-stage codegen is not supported; showing physical plan instead.
+  
+    == Plan Steps ==
+    initial_logical_plan:
+    Sort: #13 ASC NULLS FIRST
+      Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16
+        Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]]
+          Projection: #8 AS #9, #2 AS #10, #5 AS #11, #8 % CASE WHEN Int32(4) = Int32(0) THEN raise_error(Utf8("Remainder by zero")) ELSE Int32(4) END AS #12
+            Projection: #8, #2, #5
+              Projection: coalesce(#1, #4) AS #8, #1 AS #6, #2, #4 AS #7, #5
+                Inner Join: #1 = #4
+                  ExplicitRepartition: kind=hash, n=Some(8), expr=[#1]
+                    Projection: #0 AS k AS #1, #0 * Int32(2) AS v1 AS #2
+                      Range: start=0, end=64, step=1, num_partitions=4
+                  ExplicitRepartition: kind=hash, n=Some(8), expr=[#4]
+                    Projection: #3 AS k AS #4, #3 + Int32(1) AS v2 AS #5
+                      Range: start=0, end=64, step=1, num_partitions=4
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_grouping_function:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after type_coercion:
+    Sort: #13 ASC NULLS FIRST
+      Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16
+        Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]]
+          Projection: #8 AS #9, #2 AS #10, #5 AS #11, #8 % CAST(CASE WHEN Int32(4) = Int32(0) THEN CAST(raise_error(Utf8("Remainder by zero")) AS Int32) ELSE Int32(4) END AS Int64) AS #12
+            Projection: #8, #2, #5
+              Projection: coalesce(#1, #4) AS #8, #1 AS #6, #2, #4 AS #7, #5
+                Inner Join: #1 = #4
+                  ExplicitRepartition: kind=hash, n=Some(8), expr=[#1]
+                    Projection: #0 AS k AS #1, #0 * CAST(Int32(2) AS Int64) AS v1 AS #2
+                      Range: start=0, end=64, step=1, num_partitions=4
+                  ExplicitRepartition: kind=hash, n=Some(8), expr=[#4]
+                    Projection: #3 AS k AS #4, #3 + CAST(Int32(1) AS Int64) AS v2 AS #5
+                      Range: start=0, end=64, step=1, num_partitions=4
+  
+    analyzed_logical_plan:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    Sort: #13 ASC NULLS FIRST
+      Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16
+        Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]]
+          Projection: #8 AS #9, #2 AS #10, #5 AS #11, #8 % Int64(4) AS #12
+            Projection: #8, #2, #5
+              Projection: #1 AS #8, #1 AS #6, #2, #4 AS #7, #5
+                Inner Join: #1 = #4
+                  ExplicitRepartition: kind=hash, n=Some(8), expr=[#1]
+                    Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2
+                      Range: start=0, end=64, step=1, num_partitions=4
+                  ExplicitRepartition: kind=hash, n=Some(8), expr=[#4]
+                    Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5
+                      Range: start=0, end=64, step=1, num_partitions=4
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #13 ASC NULLS FIRST
+      Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16
+        Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]]
+          Projection: #2 AS #10, #5 AS #11, #8 % Int64(4) AS #12
+            Projection: #1 AS #8, #2, #5
+              Inner Join: #1 = #4
+                ExplicitRepartition: kind=hash, n=Some(8), expr=[#1]
+                  Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2
+                    Range: start=0, end=64, step=1, num_partitions=4
+                ExplicitRepartition: kind=hash, n=Some(8), expr=[#4]
+                  Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5
+                    Range: start=0, end=64, step=1, num_partitions=4
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    Sort: #13 ASC NULLS FIRST
+      Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16
+        Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]]
+          Projection: #2 AS #10, #5 AS #11, #1 % Int64(4) AS #12
+            Inner Join: #1 = #4
+              ExplicitRepartition: kind=hash, n=Some(8), expr=[#1]
+                Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2
+                  Range: start=0, end=64, step=1, num_partitions=4
+              ExplicitRepartition: kind=hash, n=Some(8), expr=[#4]
+                Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5
+                  Range: start=0, end=64, step=1, num_partitions=4
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_projection:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after rewrite_set_comparison:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_unions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after unions_to_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after simplify_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after replace_distinct_aggregate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_predicate_subquery:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after scalar_subquery_to_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after decorrelate_lateral_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_equijoin_predicate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_duplicated_expr:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_cross_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after propagate_empty_relation:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after filter_null_join_keys:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_outer_join:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_limit:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after push_down_filter:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after single_distinct_aggregation_to_group_by:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after eliminate_group_by_constant:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after common_sub_expression_eliminate:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after extract_leaf_expressions:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after optimize_projections:
+    SAME TEXT AS ABOVE
+  
+    logical_plan after resolve_lambda_variables:
+    SAME TEXT AS ABOVE
+  
+    logical_plan:
+    Sort: #13 ASC NULLS FIRST
+      Projection: #12 AS #13, count(Int64(1)) AS count AS #14, sum(#10) AS s1 AS #15, sum(#11) AS s2 AS #16
+        Aggregate: groupBy=[[#12]], aggr=[[count(Int64(1)), sum(#10), sum(#11)]]
+          Projection: #2 AS #10, #5 AS #11, #1 % Int64(4) AS #12
+            Inner Join: #1 = #4
+              ExplicitRepartition: kind=hash, n=Some(8), expr=[#1]
+                Projection: #0 AS k AS #1, #0 * Int64(2) AS v1 AS #2
+                  Range: start=0, end=64, step=1, num_partitions=4
+              ExplicitRepartition: kind=hash, n=Some(8), expr=[#4]
+                Projection: #3 AS k AS #4, #3 + Int64(1) AS v2 AS #5
+                  Range: start=0, end=64, step=1, num_partitions=4
+  
+    initial_physical_plan:
+    SortExec: expr=[#13@0 ASC], preserve_partitioning=[false]
+      ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+        AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+          AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+            ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12]
+              HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)]
+                ExplicitRepartitionExec
+                  ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                    RangeExec
+                ExplicitRepartitionExec
+                  ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                    RangeExec
+  
+  
+    initial_physical_plan_with_stats:
+    SortExec: expr=[#13@0 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]
+      ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]
+        AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]
+          AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]
+            ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]
+              HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:)]]
+                ExplicitRepartitionExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]]
+                  ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]]
+                    RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+                ExplicitRepartitionExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]]
+                  ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:)]]
+                    RangeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
+  
+  
+    initial_physical_plan_with_schema:
+    SortExec: expr=[#13@0 ASC], preserve_partitioning=[false], schema=[#13:Int64, #14:Int64, #15:Int64;N, #16:Int64;N]
+      ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16], schema=[#13:Int64, #14:Int64, #15:Int64;N, #16:Int64;N]
+        AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], schema=[#12:Int64, count(Int64(1)):Int64, sum(#10):Int64;N, sum(#11):Int64;N]
+          AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)], schema=[#12:Int64, count(Int64(1))[count]:Int64, sum(#10)[sum]:Int64;N, sum(#11)[sum]:Int64;N]
+            ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12], schema=[#10:Int64, #11:Int64, #12:Int64]
+              HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)], schema=[#1:Int64, #2:Int64, #4:Int64, #5:Int64]
+                ExplicitRepartitionExec, schema=[#1:Int64, #2:Int64]
+                  ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2], schema=[#1:Int64, #2:Int64]
+                    RangeExec, schema=[#0:Int64]
+                ExplicitRepartitionExec, schema=[#4:Int64, #5:Int64]
+                  ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5], schema=[#4:Int64, #5:Int64]
+                    RangeExec, schema=[#3:Int64]
+  
+  
+    physical_plan after OutputRequirements:
+    OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#13@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+          AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+            AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12]
+                HashJoinExec: mode=Auto, join_type=Inner, on=[(#1@0, #4@0)]
+                  ExplicitRepartitionExec
+                    ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                      RangeExec
+                  ExplicitRepartitionExec
+                    ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                      RangeExec
+  
+  
+    physical_plan after aggregate_statistics:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after join_selection:
+    OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#13@0 ASC], preserve_partitioning=[false]
+        ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+          AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+            AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12]
+                HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)]
+                  ExplicitRepartitionExec
+                    ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                      RangeExec
+                  ExplicitRepartitionExec
+                    ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                      RangeExec
+  
+  
+    physical_plan after LimitedDistinctAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceDistribution:
+    OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition
+      SortExec: expr=[#13@0 ASC], preserve_partitioning=[false]
+        CoalescePartitionsExec
+          ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+            AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+                AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                  ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12]
+                    HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)]
+                      ExplicitRepartitionExec
+                        ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                          RangeExec
+                      ExplicitRepartitionExec
+                        ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                          RangeExec
+  
+  
+    physical_plan after CombinePartialFinalAggregate:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceSorting:
+    OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#13@0 ASC]
+        SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+            AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+                AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                  ProjectionExec: expr=[#2@1 as #10, #5@3 as #11, #1@0 % 4 as #12]
+                    HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)]
+                      ExplicitRepartitionExec
+                        ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                          RangeExec
+                      ExplicitRepartitionExec
+                        ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                          RangeExec
+  
+  
+    physical_plan after OptimizeAggregateOrder:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after WindowTopN:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    OutputRequirementExec: order_by=[(#13@0, asc)], dist_by=SinglePartition
+      SortPreservingMergeExec: [#13@0 ASC]
+        SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+            AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+                AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                  ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12]
+                    HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0]
+                      ExplicitRepartitionExec
+                        ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                          RangeExec
+                      ExplicitRepartitionExec
+                        ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                          RangeExec
+  
+  
+    physical_plan after OutputRequirements:
+    SortPreservingMergeExec: [#13@0 ASC]
+      SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+          AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+            RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+              AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12]
+                  HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0]
+                    ExplicitRepartitionExec
+                      ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                        RangeExec
+                    ExplicitRepartitionExec
+                      ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                        RangeExec
+  
+  
+    physical_plan after LimitAggregation:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushPastWindows:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after HashJoinBuffering:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after LimitPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after TopKRepartition:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after ProjectionPushdown:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after PushdownSort:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnsureCooperative:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after FilterPushdown(Post):
+    SAME TEXT AS ABOVE
+  
+    physical_plan after RewriteExplicitRepartition:
+    SortPreservingMergeExec: [#13@0 ASC]
+      SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+        ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+          AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+            RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+              AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12]
+                  HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0]
+                    RepartitionExec: partitioning=Hash([#1@0], 8), input_partitions=4
+                      ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                        RangeExec
+                    RepartitionExec: partitioning=Hash([#4@0], 8), input_partitions=4
+                      ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                        RangeExec
+  
+  
+    physical_plan after RewriteCollectLeftHashJoin:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after EnforceBarrierPartitioning:
+    SAME TEXT AS ABOVE
+  
+    physical_plan after SanityCheckPlan:
+    SAME TEXT AS ABOVE
+  
+    physical_plan:
+    ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2]
+      SortPreservingMergeExec: [#13@0 ASC]
+        SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+            AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+                AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                  ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12]
+                    HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0]
+                      RepartitionExec: partitioning=Hash([#1@0], 8), input_partitions=4
+                        ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                          RangeExec
+                      RepartitionExec: partitioning=Hash([#4@0], 8), input_partitions=4
+                        ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                          RangeExec
+  
+  
+    == Physical Plan ==
+    ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2]
+      SortPreservingMergeExec: [#13@0 ASC]
+        SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+          ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+            AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+              RepartitionExec: partitioning=Hash([#12@0], 4), input_partitions=8
+                AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+                  ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12]
+                    HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0]
+                      RepartitionExec: partitioning=Hash([#1@0], 8), input_partitions=4
+                        ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+                          RangeExec
+                      RepartitionExec: partitioning=Hash([#4@0], 8), input_partitions=4
+                        ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+                          RangeExec
+  
+  
+    == Distributed Plan ==
+    === stage 0 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[#1@0], channels=8)
+    placement=Worker
+    ProjectionExec: expr=[#0@0 as #1, #0@0 * 2 as #2]
+      RangeExec
+  
+    === stage 1 ===
+    inputs=[StageInput(stage=0, mode=Shuffle)]
+    mode=Blocking
+    partitions=8
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=Hash([#1@0], 8)
+  
+    === stage 2 ===
+    inputs=[]
+    mode=Pipelined
+    partitions=4
+    distribution=Hash(keys=[#4@0], channels=8)
+    placement=Worker
+    ProjectionExec: expr=[#3@0 as #4, #3@0 + 1 as #5]
+      RangeExec
+  
+    === stage 3 ===
+    inputs=[StageInput(stage=2, mode=Shuffle)]
+    mode=Blocking
+    partitions=8
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=Hash([#4@0], 8)
+  
+    === stage 4 ===
+    inputs=[StageInput(stage=1, mode=Forward), StageInput(stage=3, mode=Forward)]
+    mode=Pipelined
+    partitions=8
+    distribution=Hash(keys=[#12@0], channels=4)
+    placement=Worker
+    AggregateExec: mode=Partial, gby=[#12@2 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+      ProjectionExec: expr=[#2@0 as #10, #5@1 as #11, #1@2 % 4 as #12]
+        HashJoinExec: mode=Partitioned, join_type=Inner, on=[(#1@0, #4@0)], projection=[#2@1, #5@3, #1@0]
+          StageInputExec: input=0, partitioning=Hash([#1@0], 8)
+          StageInputExec: input=1, partitioning=Hash([#4@0], 8)
+  
+    === stage 5 ===
+    inputs=[StageInput(stage=4, mode=Shuffle)]
+    mode=Blocking
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    StageInputExec: input=0, partitioning=Hash([#12@0], 4)
+  
+    === stage 6 ===
+    inputs=[StageInput(stage=5, mode=Forward)]
+    mode=Pipelined
+    partitions=4
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    SortExec: expr=[#13@0 ASC], preserve_partitioning=[true]
+      ProjectionExec: expr=[#12@0 as #13, count(Int64(1))@1 as #14, sum(#10)@2 as #15, sum(#11)@3 as #16]
+        AggregateExec: mode=FinalPartitioned, gby=[#12@0 as #12], aggr=[count(Int64(1)), sum(#10), sum(#11)]
+          StageInputExec: input=0, partitioning=Hash([#12@0], 4)
+  
+    === stage 7 ===
+    inputs=[StageInput(stage=6, mode=Merge)]
+    mode=Pipelined
+    partitions=1
+    distribution=RoundRobin(channels=1)
+    placement=Worker
+    ProjectionExec: expr=[#13@0 as g, #14@1 as count, #15@2 as s1, #16@3 as s2]
+      SortPreservingMergeExec: [#13@0 ASC]
+        StageInputExec: input=0, partitioning=Hash([#13@0], 4)
diff --git a/python/pysail/tests/spark/execution/conftest.py b/python/pysail/tests/spark/execution/conftest.py
new file mode 100644
index 000000000..6b9e70b34
--- /dev/null
+++ b/python/pysail/tests/spark/execution/conftest.py
@@ -0,0 +1,9 @@
+import pytest
+
+from pysail.testing.spark.session import spark_connect_server
+
+
+@pytest.fixture(scope="package")
+def remote():
+    with spark_connect_server(envs={"SAIL_MODE": "local-cluster"}) as server:
+        yield server.remote
diff --git a/python/pysail/tests/spark/execution/features/explain.feature b/python/pysail/tests/spark/execution/features/explain.feature
new file mode 100644
index 000000000..f11924950
--- /dev/null
+++ b/python/pysail/tests/spark/execution/features/explain.feature
@@ -0,0 +1,10 @@
+Feature: EXPLAIN CODEGEN in distributed execution
+  Scenario: EXPLAIN CODEGEN includes the distributed execution plan
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT k, SUM(v) AS total
+      FROM VALUES (1, 2), (1, 3) t(k, v)
+      GROUP BY k
+      """
+    Then query plan matches snapshot
diff --git a/python/pysail/tests/spark/execution/features/lakehouse_commit.feature b/python/pysail/tests/spark/execution/features/lakehouse_commit.feature
new file mode 100644
index 000000000..6c1a00524
--- /dev/null
+++ b/python/pysail/tests/spark/execution/features/lakehouse_commit.feature
@@ -0,0 +1,77 @@
+Feature: Lakehouse commits in distributed execution
+
+  Rule: Delta Lake commit execution
+    Background:
+      Given variable location for temporary directory distributed_delta_commit
+      Given final statement
+        """
+        DROP TABLE IF EXISTS distributed_delta_commit
+        """
+      Given statement template
+        """
+        CREATE TABLE distributed_delta_commit (id BIGINT)
+        USING delta
+        LOCATION {{ location.sql }}
+        """
+
+    Scenario: Delta commit runs on the driver after parallel file writing
+      When query
+        """
+        EXPLAIN CODEGEN
+        INSERT INTO distributed_delta_commit
+        SELECT id FROM range(0, 400, 1, 4)
+        """
+      Then query plan matches snapshot
+
+    Scenario: Delta file writing remains parallel on workers
+      Given statement
+        """
+        INSERT INTO distributed_delta_commit
+        SELECT id FROM range(0, 400, 1, 4)
+        """
+      Then data files in location count is 4
+      When query
+        """
+        SELECT COUNT(*) AS count FROM distributed_delta_commit
+        """
+      Then query result
+        | count |
+        | 400   |
+
+  Rule: Iceberg commit execution
+    Background:
+      Given variable location for temporary directory distributed_iceberg_commit
+      Given final statement
+        """
+        DROP TABLE IF EXISTS distributed_iceberg_commit
+        """
+      Given statement template
+        """
+        CREATE TABLE distributed_iceberg_commit (id BIGINT)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        """
+
+    Scenario: Iceberg commit runs on the driver after parallel file writing
+      When query
+        """
+        EXPLAIN CODEGEN
+        INSERT INTO distributed_iceberg_commit
+        SELECT id FROM range(0, 400, 1, 4)
+        """
+      Then query plan matches snapshot
+
+    Scenario: Iceberg file writing remains parallel on workers
+      Given statement
+        """
+        INSERT INTO distributed_iceberg_commit
+        SELECT id FROM range(0, 400, 1, 4)
+        """
+      Then data files in location count is 4
+      When query
+        """
+        SELECT COUNT(*) AS count FROM distributed_iceberg_commit
+        """
+      Then query result
+        | count |
+        | 400   |
diff --git a/python/pysail/tests/spark/execution/features/scalar_subquery.feature b/python/pysail/tests/spark/execution/features/scalar_subquery.feature
new file mode 100644
index 000000000..193c84f0c
--- /dev/null
+++ b/python/pysail/tests/spark/execution/features/scalar_subquery.feature
@@ -0,0 +1,645 @@
+Feature: Scalar subqueries in distributed execution
+  Scenario: Scalar subquery in filter before aggregate
+    When query
+      """
+      SELECT k, SUM(v) AS total
+      FROM VALUES (1, 2), (1, 3), (2, 10), (3, 1) AS t(k, v)
+      WHERE v > (SELECT MAX(x) FROM VALUES (1), (2) AS s(x))
+      GROUP BY k
+      """
+    Then query result collected
+      | k | total |
+      | 1 | 3     |
+      | 2 | 10    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT k, SUM(v) AS total
+      FROM VALUES (1, 2), (1, 3), (2, 10), (3, 1) AS t(k, v)
+      WHERE v > (SELECT MAX(x) FROM VALUES (1), (2) AS s(x))
+      GROUP BY k
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in ungrouped aggregate expression
+    When query
+      """
+      SELECT SUM(CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      )) AS total
+      FROM VALUES (1), (2), (3) AS t(v)
+      """
+    Then query result collected
+      | total |
+      | 36    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT SUM(CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      )) AS total
+      FROM VALUES (1), (2), (3) AS t(v)
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in grouped aggregate expression
+    When query
+      """
+      SELECT k, SUM(CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      )) AS total
+      FROM VALUES (1, 1), (1, 2), (2, 3) AS t(k, v)
+      GROUP BY k
+      """
+    Then query result collected
+      | k | total |
+      | 1 | 23    |
+      | 2 | 13    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT k, SUM(CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      )) AS total
+      FROM VALUES (1, 1), (1, 2), (2, 3) AS t(k, v)
+      GROUP BY k
+      """
+    Then query plan matches snapshot
+
+  Scenario: Correlated scalar subquery in filter
+    When query
+      """
+      SELECT outer_t.k, outer_t.v
+      FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v)
+      WHERE outer_t.v = (
+        SELECT MAX(inner_t.x)
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      )
+      ORDER BY outer_t.k
+      """
+    Then query result collected ordered
+      | k | v |
+      | 1 | 4 |
+      | 2 | 3 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT outer_t.k, outer_t.v
+      FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v)
+      WHERE outer_t.v = (
+        SELECT MAX(inner_t.x)
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      )
+      ORDER BY outer_t.k
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in projection before aggregate
+    When query
+      """
+      SELECT k, SUM(shifted) AS total
+      FROM (
+        SELECT k, CAST(v AS BIGINT) + (
+          SELECT MIN(CAST(x AS BIGINT))
+          FROM VALUES (10), (20) AS s(x)
+        ) AS shifted
+        FROM VALUES (1, 1), (1, 2), (2, 3) AS t(k, v)
+      ) p
+      GROUP BY k
+      """
+    Then query result collected
+      | k | total |
+      | 1 | 23    |
+      | 2 | 13    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT k, SUM(shifted) AS total
+      FROM (
+        SELECT k, CAST(v AS BIGINT) + (
+          SELECT MIN(CAST(x AS BIGINT))
+          FROM VALUES (10), (20) AS s(x)
+        ) AS shifted
+        FROM VALUES (1, 1), (1, 2), (2, 3) AS t(k, v)
+      ) p
+      GROUP BY k
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in sort expression
+    When query
+      """
+      SELECT v
+      FROM VALUES (1), (2), (3) AS t(v)
+      ORDER BY CASE WHEN (
+        SELECT MIN(x) FROM VALUES (2) AS s(x)
+      ) = 2 THEN v ELSE -v END DESC
+      """
+    Then query result collected ordered
+      | v |
+      | 3 |
+      | 2 |
+      | 1 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v
+      FROM VALUES (1), (2), (3) AS t(v)
+      ORDER BY CASE WHEN (
+        SELECT MIN(x) FROM VALUES (2) AS s(x)
+      ) = 2 THEN v ELSE -v END DESC
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in group by expression
+    When query
+      """
+      SELECT CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      ) AS g, COUNT(*) AS n
+      FROM VALUES (1), (2), (1) AS t(v)
+      GROUP BY g
+      """
+    Then query result collected
+      | g  | n |
+      | 11 | 2 |
+      | 12 | 1 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      ) AS g, COUNT(*) AS n
+      FROM VALUES (1), (2), (1) AS t(v)
+      GROUP BY g
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in join condition
+    When query
+      """
+      SELECT l.v, r.w
+      FROM VALUES (1), (2), (3) AS l(v)
+      JOIN VALUES (11), (12), (13) AS r(w)
+        ON CAST(l.v AS BIGINT) + (
+          SELECT MIN(CAST(x AS BIGINT))
+          FROM VALUES (10), (20) AS s(x)
+        ) = CAST(r.w AS BIGINT)
+      ORDER BY l.v
+      """
+    Then query result collected ordered
+      | v | w  |
+      | 1 | 11 |
+      | 2 | 12 |
+      | 3 | 13 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT l.v, r.w
+      FROM VALUES (1), (2), (3) AS l(v)
+      JOIN VALUES (11), (12), (13) AS r(w)
+        ON CAST(l.v AS BIGINT) + (
+          SELECT MIN(CAST(x AS BIGINT))
+          FROM VALUES (10), (20) AS s(x)
+        ) = CAST(r.w AS BIGINT)
+      ORDER BY l.v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in non-equi join condition
+    When query
+      """
+      SELECT l.v, r.w
+      FROM VALUES (1), (2), (3) AS l(v)
+      JOIN VALUES (11), (12), (13) AS r(w)
+        ON CAST(l.v AS BIGINT) + (
+          SELECT MIN(CAST(x AS BIGINT))
+          FROM VALUES (10), (20) AS s(x)
+        ) < CAST(r.w AS BIGINT)
+      ORDER BY l.v, r.w
+      """
+    Then query result collected ordered
+      | v | w  |
+      | 1 | 12 |
+      | 1 | 13 |
+      | 2 | 13 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT l.v, r.w
+      FROM VALUES (1), (2), (3) AS l(v)
+      JOIN VALUES (11), (12), (13) AS r(w)
+        ON CAST(l.v AS BIGINT) + (
+          SELECT MIN(CAST(x AS BIGINT))
+          FROM VALUES (10), (20) AS s(x)
+        ) < CAST(r.w AS BIGINT)
+      ORDER BY l.v, r.w
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in having filter
+    When query
+      """
+      SELECT k, SUM(v) AS total
+      FROM VALUES (1, 1), (1, 2), (2, 3), (2, 4) AS t(k, v)
+      GROUP BY k
+      HAVING SUM(v) > (SELECT MIN(x) FROM VALUES (3) AS s(x))
+      """
+    Then query result collected
+      | k | total |
+      | 2 | 7     |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT k, SUM(v) AS total
+      FROM VALUES (1, 1), (1, 2), (2, 3), (2, 4) AS t(k, v)
+      GROUP BY k
+      HAVING SUM(v) > (SELECT MIN(x) FROM VALUES (3) AS s(x))
+      """
+    Then query plan matches snapshot
+
+  Scenario: Multiple scalar subqueries in one expression
+    When query
+      """
+      SELECT SUM(CAST(v AS BIGINT) +
+        (SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x)) +
+        (SELECT MAX(CAST(y AS BIGINT)) FROM VALUES (1), (2) AS u(y))
+      ) AS total
+      FROM VALUES (1), (2), (3) AS t(v)
+      """
+    Then query result collected
+      | total |
+      | 42    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT SUM(CAST(v AS BIGINT) +
+        (SELECT MIN(CAST(x AS BIGINT)) FROM VALUES (10), (20) AS s(x)) +
+        (SELECT MAX(CAST(y AS BIGINT)) FROM VALUES (1), (2) AS u(y))
+      ) AS total
+      FROM VALUES (1), (2), (3) AS t(v)
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in window order expression
+    When query
+      """
+      SELECT v, ROW_NUMBER() OVER (ORDER BY CASE WHEN (
+        SELECT MIN(x) FROM VALUES (2) AS s(x)
+      ) = 2 THEN v ELSE -v END) AS rn
+      FROM VALUES (3), (1), (2) AS t(v)
+      ORDER BY rn
+      """
+    Then query result collected ordered
+      | v | rn |
+      | 1 | 1  |
+      | 2 | 2  |
+      | 3 | 3  |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v, ROW_NUMBER() OVER (ORDER BY CASE WHEN (
+        SELECT MIN(x) FROM VALUES (2) AS s(x)
+      ) = 2 THEN v ELSE -v END) AS rn
+      FROM VALUES (3), (1), (2) AS t(v)
+      ORDER BY rn
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in window partition expression
+    When query
+      """
+      SELECT v, COUNT(*) OVER (PARTITION BY CASE WHEN (
+        SELECT MIN(x) FROM VALUES (1) AS s(x)
+      ) = 1 THEN v % 2 ELSE v END) AS n
+      FROM VALUES (1), (2), (3), (4) AS t(v)
+      ORDER BY v
+      """
+    Then query result collected ordered
+      | v | n |
+      | 1 | 2 |
+      | 2 | 2 |
+      | 3 | 2 |
+      | 4 | 2 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v, COUNT(*) OVER (PARTITION BY CASE WHEN (
+        SELECT MIN(x) FROM VALUES (1) AS s(x)
+      ) = 1 THEN v % 2 ELSE v END) AS n
+      FROM VALUES (1), (2), (3), (4) AS t(v)
+      ORDER BY v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in window aggregate argument
+    When query
+      """
+      SELECT v, SUM(CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      )) OVER () AS total
+      FROM VALUES (1), (2), (3) AS t(v)
+      ORDER BY v
+      """
+    Then query result collected ordered
+      | v | total |
+      | 1 | 36    |
+      | 2 | 36    |
+      | 3 | 36    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v, SUM(CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (10), (20) AS s(x)
+      )) OVER () AS total
+      FROM VALUES (1), (2), (3) AS t(v)
+      ORDER BY v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Scalar subquery in aggregate filter
+    When query
+      """
+      SELECT k, COUNT(*) FILTER (WHERE v > (
+        SELECT MIN(x) FROM VALUES (1) AS s(x)
+      )) AS n
+      FROM VALUES (1, 1), (1, 2), (2, 3) AS t(k, v)
+      GROUP BY k
+      """
+    Then query result collected
+      | k | n |
+      | 1 | 1 |
+      | 2 | 1 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT k, COUNT(*) FILTER (WHERE v > (
+        SELECT MIN(x) FROM VALUES (1) AS s(x)
+      )) AS n
+      FROM VALUES (1, 1), (1, 2), (2, 3) AS t(k, v)
+      GROUP BY k
+      """
+    Then query plan matches snapshot
+
+  @spark-4
+  Scenario: Scalar subquery in aggregate order by
+    When query
+      """
+      SELECT listagg(v, ',') WITHIN GROUP (
+        ORDER BY CASE WHEN (
+          SELECT MIN(x) FROM VALUES (1) AS s(x)
+        ) = 1 THEN sort_key ELSE -sort_key END
+      ) AS joined
+      FROM VALUES ('a', 2), ('b', 1), ('c', 3) AS t(v, sort_key)
+      """
+    Then query result collected
+      | joined |
+      | b,a,c  |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT listagg(v, ',') WITHIN GROUP (
+        ORDER BY CASE WHEN (
+          SELECT MIN(x) FROM VALUES (1) AS s(x)
+        ) = 1 THEN sort_key ELSE -sort_key END
+      ) AS joined
+      FROM VALUES ('a', 2), ('b', 1), ('c', 3) AS t(v, sort_key)
+      """
+    Then query plan matches snapshot
+
+  Scenario: Nested scalar subquery
+    When query
+      """
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT MIN(x) + (SELECT MIN(y) FROM VALUES (1) AS u(y))
+        FROM VALUES (10), (20) AS s(x)
+      ) AS shifted
+      FROM VALUES (1), (2) AS t(v)
+      ORDER BY v
+      """
+    Then query result collected ordered
+      | v | shifted |
+      | 1 | 12      |
+      | 2 | 13      |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT MIN(x) + (SELECT MIN(y) FROM VALUES (1) AS u(y))
+        FROM VALUES (10), (20) AS s(x)
+      ) AS shifted
+      FROM VALUES (1), (2) AS t(v)
+      ORDER BY v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Empty scalar subquery result
+    When query
+      """
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT CAST(x AS BIGINT)
+        FROM VALUES (10) AS s(x)
+        WHERE x > 100
+      ) AS shifted
+      FROM VALUES (1), (2) AS t(v)
+      ORDER BY v
+      """
+    Then query result collected ordered
+      | v | shifted |
+      | 1 | NULL    |
+      | 2 | NULL    |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT CAST(x AS BIGINT)
+        FROM VALUES (10) AS s(x)
+        WHERE x > 100
+      ) AS shifted
+      FROM VALUES (1), (2) AS t(v)
+      ORDER BY v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Null scalar subquery result
+    When query
+      """
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (CAST(NULL AS INT)) AS s(x)
+      ) AS maybe_shifted
+      FROM VALUES (1), (2) AS t(v)
+      ORDER BY v
+      """
+    Then query result collected ordered
+      | v | maybe_shifted |
+      | 1 | NULL          |
+      | 2 | NULL          |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT MIN(CAST(x AS BIGINT))
+        FROM VALUES (CAST(NULL AS INT)) AS s(x)
+      ) AS maybe_shifted
+      FROM VALUES (1), (2) AS t(v)
+      ORDER BY v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Multi-row scalar subquery errors
+    When query
+      """
+      SELECT v, CAST(v AS BIGINT) + (
+        SELECT CAST(x AS BIGINT)
+        FROM VALUES (10), (20) AS s(x)
+      ) AS shifted
+      FROM VALUES (1) AS t(v)
+      """
+    Then query error (?i)(SCALAR_SUBQUERY_TOO_MANY_ROWS|more than one row)
+
+  # Spark 4 analyzes this query and raises the scalar-subquery cardinality error
+  # at runtime. Sail currently rejects it during planning via DataFusion's
+  # correlated scalar subquery invariant.
+  @spark-4
+  @sail-bug
+  Scenario: Unaggregated correlated scalar subquery errors at runtime
+    When query
+      """
+      SELECT outer_t.k, (
+        SELECT inner_t.x
+        FROM VALUES (1, 4), (1, 2), (2, 3) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS x
+      FROM VALUES (1), (2) AS outer_t(k)
+      ORDER BY outer_t.k
+      """
+    Then query error (?i)(SCALAR_SUBQUERY_TOO_MANY_ROWS|more than one row)
+
+  Scenario: Correlated scalar subquery in projection
+    When query
+      """
+      SELECT outer_t.k, outer_t.v, (
+        SELECT MAX(inner_t.x)
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS mx
+      FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v)
+      ORDER BY outer_t.k, outer_t.v
+      """
+    Then query result collected ordered
+      | k | v | mx |
+      | 1 | 2 | 4  |
+      | 1 | 4 | 4  |
+      | 2 | 1 | 3  |
+      | 2 | 3 | 3  |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT outer_t.k, outer_t.v, (
+        SELECT MAX(inner_t.x)
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS mx
+      FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v)
+      ORDER BY outer_t.k, outer_t.v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Correlated scalar subquery no-match result
+    When query
+      """
+      SELECT outer_t.k, outer_t.v, (
+        SELECT MAX(inner_t.x)
+        FROM VALUES (1, 4), (1, 2) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS mx
+      FROM VALUES (1, 2), (2, 3) AS outer_t(k, v)
+      ORDER BY outer_t.k
+      """
+    Then query result collected ordered
+      | k | v | mx   |
+      | 1 | 2 | 4    |
+      | 2 | 3 | NULL |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT outer_t.k, outer_t.v, (
+        SELECT MAX(inner_t.x)
+        FROM VALUES (1, 4), (1, 2) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS mx
+      FROM VALUES (1, 2), (2, 3) AS outer_t(k, v)
+      ORDER BY outer_t.k
+      """
+    Then query plan matches snapshot
+
+  Scenario: Correlated scalar subquery with nested scalar in projection
+    When query
+      """
+      SELECT outer_t.k, outer_t.v, (
+        SELECT MAX(inner_t.x) + (SELECT MIN(y) FROM VALUES (1) AS u(y))
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS shifted
+      FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v)
+      ORDER BY outer_t.k, outer_t.v
+      """
+    Then query result collected ordered
+      | k | v | shifted |
+      | 1 | 2 | 5       |
+      | 1 | 4 | 5       |
+      | 2 | 1 | 4       |
+      | 2 | 3 | 4       |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT outer_t.k, outer_t.v, (
+        SELECT MAX(inner_t.x) + (SELECT MIN(y) FROM VALUES (1) AS u(y))
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      ) AS shifted
+      FROM VALUES (1, 2), (1, 4), (2, 1), (2, 3) AS outer_t(k, v)
+      ORDER BY outer_t.k, outer_t.v
+      """
+    Then query plan matches snapshot
+
+  Scenario: Correlated scalar subquery with nested scalar in filter
+    When query
+      """
+      SELECT outer_t.k, outer_t.v
+      FROM VALUES (1, 2), (1, 5), (2, 1), (2, 4) AS outer_t(k, v)
+      WHERE outer_t.v = (
+        SELECT MAX(inner_t.x) + (SELECT MIN(y) FROM VALUES (1) AS u(y))
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      )
+      ORDER BY outer_t.k
+      """
+    Then query result collected ordered
+      | k | v |
+      | 1 | 5 |
+      | 2 | 4 |
+    When query
+      """
+      EXPLAIN CODEGEN
+      SELECT outer_t.k, outer_t.v
+      FROM VALUES (1, 2), (1, 5), (2, 1), (2, 4) AS outer_t(k, v)
+      WHERE outer_t.v = (
+        SELECT MAX(inner_t.x) + (SELECT MIN(y) FROM VALUES (1) AS u(y))
+        FROM VALUES (1, 4), (1, 2), (2, 3), (2, 1) AS inner_t(k, x)
+        WHERE inner_t.k = outer_t.k
+      )
+      ORDER BY outer_t.k
+      """
+    Then query plan matches snapshot
diff --git a/python/pysail/tests/spark/execution/test_driver_server.py b/python/pysail/tests/spark/execution/test_driver_server.py
new file mode 100644
index 000000000..1aec34669
--- /dev/null
+++ b/python/pysail/tests/spark/execution/test_driver_server.py
@@ -0,0 +1,60 @@
+import socket
+
+import pytest
+
+from pysail.spark import SparkConnectServer
+from pysail.testing.spark.session import spark_connect_server, spark_session_factory
+
+pytestmark = pytest.mark.skip(reason="flaky tests that should be run manually")
+
+
+@pytest.fixture(scope="module")
+def remote():
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        _, driver_port = sock.getsockname()
+    envs = {
+        "SAIL_MODE": "local-cluster",
+        "SAIL_CLUSTER__DRIVER_LISTEN_PORT": str(driver_port),
+    }
+    with spark_connect_server(envs=envs) as server:
+        yield server.remote
+
+
+def test_multiple_sessions_share_driver_server(spark, remote):
+    with spark_session_factory(remote) as registry:
+        other = registry.create()
+
+        assert spark.sql("SELECT 1 AS value").first().value == 1
+        assert other.sql("SELECT 2 AS value").first().value == 2  # noqa: PLR2004
+
+
+def test_driver_gateway_startup_failure(monkeypatch):
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        _, driver_port = sock.getsockname()
+        monkeypatch.setenv("SAIL_MODE", "local-cluster")
+        monkeypatch.setenv("SAIL_CLUSTER__DRIVER_LISTEN_PORT", str(driver_port))
+
+        server = SparkConnectServer("127.0.0.1", 0)
+        with pytest.raises(RuntimeError, match="failed to create driver gateway"):
+            server.start(background=False)
+
+
+def test_session_manager_stops_with_server(monkeypatch):
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        _, driver_port = sock.getsockname()
+    monkeypatch.setenv("SAIL_MODE", "local-cluster")
+    monkeypatch.setenv("SAIL_CLUSTER__DRIVER_LISTEN_PORT", str(driver_port))
+
+    server = SparkConnectServer("127.0.0.1", 0)
+    server.start(background=True)
+    _, port = server.listening_address
+    with spark_session_factory(f"sc://localhost:{port}") as registry:
+        spark = registry.create()
+        assert spark.sql("SELECT 1 AS value").first().value == 1
+    server.stop()
+
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", driver_port))
diff --git a/python/pysail/tests/spark/execution/test_execution.py b/python/pysail/tests/spark/execution/test_execution.py
index e0636406e..8989c6a9c 100644
--- a/python/pysail/tests/spark/execution/test_execution.py
+++ b/python/pysail/tests/spark/execution/test_execution.py
@@ -5,7 +5,6 @@
 from pyspark.sql.types import Row
 from pyspark.sql.window import Window
 
-from pysail.testing.spark.session import spark_connect_server
 from pysail.testing.spark.utils.common import is_jvm_spark
 
 pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail local-cluster mode only")
@@ -18,13 +17,6 @@ def counter(_iterator):
     return df.mapInPandas(counter, schema="n: long").count()
 
 
-@pytest.fixture(scope="module")
-def remote():
-    """Override the global remote fixture to use local-cluster mode for this module."""
-    with spark_connect_server(envs={"SAIL_MODE": "local-cluster"}) as server:
-        yield server.remote
-
-
 @pytest.fixture(scope="module")
 def large_dataset(spark):
     """Create a larger dataset to test distributed execution."""
@@ -232,6 +224,16 @@ def test_repartitioning_in_cluster_mode(large_dataset):
     assert coalesced_sum == original_sum
 
 
+def test_parquet_directory_scan_reads_each_file_once_in_cluster_mode(spark, tmp_path):
+    path = tmp_path / "parquet_files"
+    path.mkdir()
+    pd.DataFrame({"id": [1]}).to_parquet(path / "part-0.parquet")
+    pd.DataFrame({"id": [2]}).to_parquet(path / "part-1.parquet")
+
+    rows = spark.read.parquet(str(path)).orderBy("id").collect()
+    assert rows == [Row(id=1), Row(id=2)]
+
+
 def test_coalesce_plan_contains_dedicated_exec_in_cluster_mode(spark):
     plan = spark.range(0, 12, 1, 4).coalesce(2)._explain_string()  # noqa: SLF001
     assert "CoalesceExec" in plan
diff --git a/python/pysail/tests/spark/execution/test_shuffle_storage.py b/python/pysail/tests/spark/execution/test_shuffle_storage.py
new file mode 100644
index 000000000..df3a0a470
--- /dev/null
+++ b/python/pysail/tests/spark/execution/test_shuffle_storage.py
@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import pandas as pd
+import pyspark.sql.functions as F  # noqa: N812
+import pytest
+from pandas.testing import assert_frame_equal
+from pyspark.sql.types import Row
+
+from pysail.testing.spark.session import spark_connect_server
+from pysail.testing.spark.steps.plan import normalize_plan_text
+from pysail.testing.spark.utils.common import is_jvm_spark
+
+pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail local-cluster mode only")
+
+
+@pytest.fixture(scope="module")
+def remote(tmp_path_factory):
+    shuffle_path = tmp_path_factory.mktemp("shuffle_storage")
+    envs = {
+        "SAIL_MODE": "local-cluster",
+        "SAIL_CLUSTER__SHUFFLE_BACKEND__TYPE": "storage",
+        "SAIL_CLUSTER__SHUFFLE_BACKEND__STORAGE__PATH": shuffle_path.as_uri(),
+        "SAIL_CLUSTER__SHUFFLE_BACKEND__STORAGE__MAX_FILE_SIZE": "1024",
+        "SAIL_CLUSTER__SHUFFLE_BACKEND__STORAGE__COMPRESSION": "lz4",
+    }
+    with spark_connect_server(envs=envs) as server:
+        yield server.remote
+
+
+@pytest.mark.yamlsnapshot(group="plan")
+def test_query_execution_with_storage_shuffle(spark, snapshot):
+    left = spark.range(0, 64, 1, 4).select(
+        F.col("id").alias("k"),
+        (F.col("id") * 2).alias("v1"),
+    )
+    right = spark.range(0, 64, 1, 4).select(
+        F.col("id").alias("k"),
+        (F.col("id") + 1).alias("v2"),
+    )
+
+    df = (
+        left.repartition(8, "k")
+        .join(right.repartition(8, "k"), "k")
+        .withColumn("g", F.col("k") % 4)
+        .groupBy("g")
+        .agg(
+            F.count("*").alias("count"),
+            F.sum("v1").alias("s1"),
+            F.sum("v2").alias("s2"),
+        )
+        .orderBy("g")
+    )
+
+    plan = normalize_plan_text(df._explain_string(mode="codegen"))  # noqa: SLF001
+    assert plan == snapshot
+
+    actual = df.toPandas()
+    expected = pd.DataFrame(
+        {
+            "g": [0, 1, 2, 3],
+            "count": [16, 16, 16, 16],
+            "s1": [960, 992, 1024, 1056],
+            "s2": [496, 512, 528, 544],
+        }
+    ).astype(
+        {
+            "g": "int64",
+            "count": "int64",
+            "s1": "int64",
+            "s2": "int64",
+        }
+    )
+    assert_frame_equal(actual, expected)
+
+
+def test_repartition_collect_with_storage_shuffle(spark):
+    rows = (
+        spark.createDataFrame([Row(id=i, group=i % 3) for i in range(30)])
+        .repartition(6, "group")
+        .groupBy("group")
+        .count()
+        .orderBy("group")
+        .collect()
+    )
+
+    assert rows == [
+        Row(group=0, count=10),
+        Row(group=1, count=10),
+        Row(group=2, count=10),
+    ]
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/concat_simplify.yaml b/python/pysail/tests/spark/function/__snapshots__/features/concat_simplify.yaml
deleted file mode 100644
index 2971d8d47..000000000
--- a/python/pysail/tests/spark/function/__snapshots__/features/concat_simplify.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-# serializer version: 1
----
-name: "test_explain_concat_of_single_string_column__no_spark_concat_in_plan"
-data:
-  |-
-    == Physical Plan ==
-    ProjectionExec: expr=[#2@0 as result]
-      ProjectionExec: expr=[column1@0 as #2]
-        DataSourceExec: partitions=1, partition_sizes=[1]
----
-name: "test_explain_concat_of_single_integer_column__spark_concat_stays_in_plan"
-data:
-  |-
-    == Physical Plan ==
-    ProjectionExec: expr=[#2@0 as result]
-      ProjectionExec: expr=[spark_concat(column1@0) as #2]
-        DataSourceExec: partitions=1, partition_sizes=[1]
----
-name: "test_explain_concat_of_two_array_columns__spark_concat_stays_in_plan"
-data:
-  |-
-    == Physical Plan ==
-    ProjectionExec: expr=[#4@0 as result]
-      ProjectionExec: expr=[spark_concat(column1@0, column2@1) as #4]
-        DataSourceExec: partitions=1, partition_sizes=[1]
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/last_day.yaml b/python/pysail/tests/spark/function/__snapshots__/features/datetime/last_day.yaml
similarity index 100%
rename from python/pysail/tests/spark/function/__snapshots__/features/last_day.yaml
rename to python/pysail/tests/spark/function/__snapshots__/features/datetime/last_day.yaml
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/year.yaml b/python/pysail/tests/spark/function/__snapshots__/features/datetime/year.yaml
similarity index 100%
rename from python/pysail/tests/spark/function/__snapshots__/features/year.yaml
rename to python/pysail/tests/spark/function/__snapshots__/features/datetime/year.yaml
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/generator/variant_explode.yaml b/python/pysail/tests/spark/function/__snapshots__/features/generator/variant_explode.yaml
new file mode 100644
index 000000000..9fe2b2a96
--- /dev/null
+++ b/python/pysail/tests/spark/function/__snapshots__/features/generator/variant_explode.yaml
@@ -0,0 +1,23 @@
+# serializer version: 1
+---
+name: "test_explain_variant_explode_outer_shows_physical_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#8@0 as pos, #9@1 as key, #10@2 as value]
+      ProjectionExec: expr=[#2@0 as #8, #3@1 as #9, #4@2 as #10]
+        UnnestExec
+          ProjectionExec: expr=[array_struct_field(__common_expr_1@0, pos) as #2, array_struct_field(__common_expr_1@0, key) as #3, array_struct_field(__common_expr_1@0, value) as #4]
+            ProjectionExec: expr=[spark_variant_explode({value:02010000020c01,metadata:1101000161}) as __common_expr_1]
+              PlaceholderRowExec
+---
+name: "test_explain_variant_explode_shows_physical_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#8@0 as pos, #9@1 as key, #10@2 as value]
+      ProjectionExec: expr=[#2@0 as #8, #3@1 as #9, #4@2 as #10]
+        UnnestExec
+          ProjectionExec: expr=[array_struct_field(__common_expr_1@0, pos) as #2, array_struct_field(__common_expr_1@0, key) as #3, array_struct_field(__common_expr_1@0, value) as #4]
+            ProjectionExec: expr=[spark_variant_explode({value:03020002040c010c02,metadata:010000}) as __common_expr_1]
+              PlaceholderRowExec
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/abs.yaml b/python/pysail/tests/spark/function/__snapshots__/features/math/abs.yaml
similarity index 100%
rename from python/pysail/tests/spark/function/__snapshots__/features/abs.yaml
rename to python/pysail/tests/spark/function/__snapshots__/features/math/abs.yaml
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/ceil_floor_preimage.yaml b/python/pysail/tests/spark/function/__snapshots__/features/math/ceil_floor_preimage.yaml
similarity index 100%
rename from python/pysail/tests/spark/function/__snapshots__/features/ceil_floor_preimage.yaml
rename to python/pysail/tests/spark/function/__snapshots__/features/math/ceil_floor_preimage.yaml
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/ceil_floor_simplify.yaml b/python/pysail/tests/spark/function/__snapshots__/features/math/ceil_floor_simplify.yaml
similarity index 100%
rename from python/pysail/tests/spark/function/__snapshots__/features/ceil_floor_simplify.yaml
rename to python/pysail/tests/spark/function/__snapshots__/features/math/ceil_floor_simplify.yaml
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/string/concat_simplify.yaml b/python/pysail/tests/spark/function/__snapshots__/features/string/concat_simplify.yaml
new file mode 100644
index 000000000..8adcbc2f5
--- /dev/null
+++ b/python/pysail/tests/spark/function/__snapshots__/features/string/concat_simplify.yaml
@@ -0,0 +1,73 @@
+# serializer version: 1
+---
+name: "test_explain_allliteral_concat_has_no_spark_concat_in_the_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#0@0 as result]
+      ProjectionExec: expr=[abcdef as #0]
+        PlaceholderRowExec
+---
+name: "test_explain_concat_of_single_integer_column__spark_concat_stays_in_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[spark_concat(column1@0) as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_concat_of_single_string_column__no_spark_concat_in_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[column1@0 as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_concat_of_two_array_columns__spark_concat_stays_in_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#4@0 as result]
+      ProjectionExec: expr=[spark_concat(column1@0, column2@1) as #4]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_concat_with_a_null_literal_drops_an_expensive_argument"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[NULL as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_deeply_nested_concat_over_columns_flattens_to_one_spark_concat"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#8@0 as result]
+      ProjectionExec: expr=[spark_concat(column1@0, column2@1, column3@2, column4@3) as #8]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_inner_null_folds_the_flattened_concat_and_drops_abs"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as result]
+      ProjectionExec: expr=[NULL as #6]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_nested_concat_flattens_and_keeps_the_abs_argument"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as result]
+      ProjectionExec: expr=[spark_concat(column1@0, column2@1, CAST(spark_abs(column3@2) AS Utf8)) as #6]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_nested_concat_over_columns_flattens_to_one_spark_concat"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#6@0 as result]
+      ProjectionExec: expr=[spark_concat(column1@0, column2@1, column3@2) as #6]
+        DataSourceExec: partitions=1, partition_sizes=[1]
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/string/concat_ws_simplify.yaml b/python/pysail/tests/spark/function/__snapshots__/features/string/concat_ws_simplify.yaml
new file mode 100644
index 000000000..4643f39eb
--- /dev/null
+++ b/python/pysail/tests/spark/function/__snapshots__/features/string/concat_ws_simplify.yaml
@@ -0,0 +1,41 @@
+# serializer version: 1
+---
+name: "test_explain_columnonly_separator__spark_concat_ws_stays_in_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[spark_concat_ws(column1@0) as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_nested_nullseparator_fold_over_a_column_collapses_to_a_constant"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[b as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_null_separator_drops_an_expensive_value_argument"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[NULL as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_null_separator_literal_over_a_column__no_spark_concat_ws_in_plan"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[NULL as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
+---
+name: "test_explain_null_value_literal_trailing_a_column__spark_concat_ws_stays_null_pruned"
+data:
+  |-
+    == Physical Plan ==
+    ProjectionExec: expr=[#2@0 as result]
+      ProjectionExec: expr=[spark_concat_ws(,, column1@0) as #2]
+        DataSourceExec: partitions=1, partition_sizes=[1]
diff --git a/python/pysail/tests/spark/function/__snapshots__/features/variant_explode.yaml b/python/pysail/tests/spark/function/__snapshots__/features/variant_explode.yaml
deleted file mode 100644
index 52574bcef..000000000
--- a/python/pysail/tests/spark/function/__snapshots__/features/variant_explode.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-# serializer version: 1
----
-name: "test_explain_variant_explode_shows_physical_plan"
-data:
-  |-
-    == Physical Plan ==
-    ProjectionExec: expr=[#5@0 as pos, #6@1 as key, #7@2 as value]
-      ProjectionExec: expr=[get_field(#1@0, pos) as #5, get_field(#1@0, key) as #6, get_field(#1@0, value) as #7]
-        UnnestExec
-          ProjectionExec: expr=[spark_variant_explode({metadata:010000,value:03020002040c010c02}) as #1]
-            PlaceholderRowExec
----
-name: "test_explain_variant_explode_outer_shows_physical_plan"
-data:
-  |-
-    == Physical Plan ==
-    ProjectionExec: expr=[#5@0 as pos, #6@1 as key, #7@2 as value]
-      ProjectionExec: expr=[get_field(#1@0, pos) as #5, get_field(#1@0, key) as #6, get_field(#1@0, value) as #7]
-        UnnestExec
-          ProjectionExec: expr=[spark_variant_explode({metadata:1101000161,value:02010000020c01}) as #1]
-            PlaceholderRowExec
\ No newline at end of file
diff --git a/python/pysail/tests/spark/function/aggregate/test_max_by_min_by.txt b/python/pysail/tests/spark/function/aggregate/test_max_by_min_by.txt
deleted file mode 100644
index 79ce33ec2..000000000
--- a/python/pysail/tests/spark/function/aggregate/test_max_by_min_by.txt
+++ /dev/null
@@ -1,123 +0,0 @@
-# BASE
->>> spark.sql("""
-... CREATE OR REPLACE TEMP VIEW alltypes AS
-... SELECT
-...   CAST(v AS INT) AS int_col,
-...   CAST(v % 3 AS BIGINT) AS bigint_col
-... FROM (SELECT explode(sequence(0, 9)) AS v)
-... """)
-DataFrame[value: boolean]
-
->>> spark.sql("""
-... CREATE OR REPLACE TEMP VIEW t_base AS
-... SELECT
-...   int_col,
-...   bigint_col,
-...   CASE WHEN int_col IN (0, 9) THEN NULL ELSE int_col END AS by_col,
-...   int_col * 10 AS val_col
-... FROM alltypes
-... """)
-DataFrame[value: boolean]
-
->>> spark.sql("SELECT * FROM t_base").show()
-+-------+----------+------+-------+
-|int_col|bigint_col|by_col|val_col|
-+-------+----------+------+-------+
-|      0|         0|  NULL|      0|
-|      1|         1|     1|     10|
-|      2|         2|     2|     20|
-|      3|         0|     3|     30|
-|      4|         1|     4|     40|
-|      5|         2|     5|     50|
-|      6|         0|     6|     60|
-|      7|         1|     7|     70|
-|      8|         2|     8|     80|
-|      9|         0|  NULL|     90|
-+-------+----------+------+-------+
-
-# 1) argmin, unfiltered, non-null  -> expected: 10
->>> spark.sql("SELECT min_by(val_col, by_col) AS result FROM t_base").show()
-+------+
-|result|
-+------+
-|    10|
-+------+
-
-# 2) argmin, unfiltered, null_result=True  -> expected: NULL
->>> spark.sql("""
-... SELECT min_by(val_col,
-...               CASE WHEN by_col IS NULL THEN by_col ELSE by_col END) AS result
-... FROM (SELECT CASE WHEN val_col = 10 THEN NULL ELSE val_col END AS val_col, by_col FROM t_base)
-... """).show()
-+------+
-|result|
-+------+
-|  NULL|
-+------+
-
-# 3) argmin, filtered (excluye int_col=1), non-null  -> expected: 20
->>> spark.sql("""
-... SELECT min_by(val_col, by_col) AS result
-... FROM t_base
-... WHERE int_col <> 1
-... """).show()
-+------+
-|result|
-+------+
-|    20|
-+------+
-
->>> # 4) argmin, filtered + null_result=True -> expected: NULL
->>> spark.sql("""
-... SELECT min_by(val_col, by_col) AS result
-... FROM (SELECT CASE WHEN val_col = 20 THEN NULL ELSE val_col END AS val_col, by_col, int_col FROM t_base)
-... WHERE int_col <> 1
-... """).show()
-+------+
-|result|
-+------+
-|  NULL|
-+------+
-
-# 5) argmax, unfiltered, non-null  -> expected: 80
->>> spark.sql("SELECT max_by(val_col, by_col) AS result FROM t_base").show()
-+------+
-|result|
-+------+
-|    80|
-+------+
-
->>> # 6) argmax, unfiltered, null_result=True  -> expected: NULL
->>> spark.sql("""
-... SELECT max_by(val_col, by_col) AS result
-... FROM (SELECT CASE WHEN val_col = 80 THEN NULL ELSE val_col END AS val_col, by_col FROM t_base)
-... """).show()
-+------+
-|result|
-+------+
-|  NULL|
-+------+
-
-# 7) argmax, filtered (excluye int_col=8), non-null  -> expected: 70
->>> spark.sql("""
-... SELECT max_by(val_col, by_col) AS result
-... FROM t_base
-... WHERE int_col <> 8
-... """).show()
-+------+
-|result|
-+------+
-|    70|
-+------+
-
->>> # 8) argmax, filtered + null_result=True -> expected: NULL
->>> spark.sql("""
-... SELECT max_by(val_col, by_col) AS result
-... FROM (SELECT CASE WHEN val_col = 70 THEN NULL ELSE val_col END AS val_col, by_col, int_col FROM t_base)
-... WHERE int_col <> 8
-... """).show()
-+------+
-|result|
-+------+
-|  NULL|
-+------+
diff --git a/python/pysail/tests/spark/function/aggregate/test_mode.txt b/python/pysail/tests/spark/function/aggregate/test_mode.txt
deleted file mode 100644
index d9ff9e020..000000000
--- a/python/pysail/tests/spark/function/aggregate/test_mode.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-# 1) mode(col), unique most frequent value -> expected: 10
->>> spark.sql("SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col);").show()
-+---------+
-|mode(col)|
-+---------+
-|       10|
-+---------+
-
-# 2) WITHIN GROUP ascending, frequency tie between 10 and 20 -> expected: lowest (10)
-# Spark names the column after its internal reverse flag, so the ascending query
-# is displayed as DESC.
->>> spark.sql("SELECT mode() WITHIN GROUP (ORDER BY col) FROM VALUES (0), (10), (10), (20), (20) AS tab(col);").show()
-+---------------------------------------+
-|mode() WITHIN GROUP (ORDER BY col DESC)|
-+---------------------------------------+
-|                                     10|
-+---------------------------------------+
-
-# 3) WITHIN GROUP ascending, unique most frequent value -> expected: 10
->>> spark.sql("SELECT mode() WITHIN GROUP (ORDER BY col) FROM VALUES (0), (10), (10) AS tab(col);").show()
-+---------------------------------------+
-|mode() WITHIN GROUP (ORDER BY col DESC)|
-+---------------------------------------+
-|                                     10|
-+---------------------------------------+
-
-# 4) WITHIN GROUP descending, frequency tie between 10 and 20 -> expected: highest (20)
-# The descending query is displayed without a sort direction.
->>> spark.sql("SELECT mode() WITHIN GROUP (ORDER BY col DESC) FROM VALUES (0), (10), (10), (20), (20) AS tab(col);").show()
-+----------------------------------+
-|mode() WITHIN GROUP (ORDER BY col)|
-+----------------------------------+
-|                                20|
-+----------------------------------+
diff --git a/python/pysail/tests/spark/function/aggregate/test_try_avg.txt b/python/pysail/tests/spark/function/aggregate/test_try_avg.txt
deleted file mode 100644
index 6d9251b4f..000000000
--- a/python/pysail/tests/spark/function/aggregate/test_try_avg.txt
+++ /dev/null
@@ -1,231 +0,0 @@
->>> from pyspark.sql import functions as sf
->>> from pyspark.sql.types import LongType
-
->>> df = spark.createDataFrame([(1,), (2,), (3,)], ["x"])
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-|  2.0|
-+-----+
-
->>> df.select(sf.try_avg("x").alias("avg_x")).printSchema()
-root
- |-- avg_x: double (nullable = true)
-
->>> df = spark.createDataFrame([(None,), (2,), (None,)], ["x"])
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-|  2.0|
-+-----+
-
->>> LONG_MAX = 2**63 - 1
->>> df = (
-...     spark.createDataFrame([(LONG_MAX,), (1,)], ["x"])
-...       .withColumn("x", sf.col("x").cast(LongType()))
-... )
->>> df.select(sf.try_avg("x").alias("avg_x")).show(truncate=False)
-+--------------------+
-|avg_x               |
-+--------------------+
-|4.611686018427388e18|
-+--------------------+
-
->>> df.select(sf.try_avg("x").alias("avg_x")).show(truncate=False)
-+--------------------+
-|avg_x               |
-+--------------------+
-|4.611686018427388e18|
-+--------------------+
-
->>> df = spark.createDataFrame([(1.5,), (2.5,), (3.0,)], ["x"])
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+------------------+
-|             avg_x|
-+------------------+
-|2.3333333333333335|
-+------------------+
-
-
->>> df = spark.createDataFrame([(1e308,), (1e308,)], ["x"])
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+--------+
-|   avg_x|
-+--------+
-|Infinity|
-+--------+
-
->>> df = spark.createDataFrame([(float('nan'),), (1.0,)], ["x"])
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-|  NaN|
-+-----+
-
->>> from decimal import Decimal
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame(
-...     [(Decimal("1.23"),), (Decimal("4.77"),)],
-...     "x DECIMAL(10,2)"
-... )
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-| 3.00|
-+-----+
-
-# pyspark
-# +--------+
-# |   avg_x|
-# +--------+
-# |3.000000|
-# +--------+
-
->>> df = spark.createDataFrame(
-...     [(Decimal("1.00"),), (None,), (Decimal("2.50"),)],
-...     "x DECIMAL(10,2)"
-... )
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-| 1.75|
-+-----+
-
-# pyspark
-# +--------+
-# |   avg_x|
-# +--------+
-# |1.750000|
-# +--------+
-
->>> from decimal import Decimal
->>> df = spark.createDataFrame(
-...     [(Decimal("90000"),), (Decimal("20000"),)],
-...     "x DECIMAL(5,0)"
-... )
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-|55000|
-+-----+
-
-# pyspark
-# +----------+
-# |     avg_x|
-# +----------+
-# |55000.0000|
-# +----------+
-
->>> from pyspark.sql import functions as sf
->>> from decimal import Decimal
->>> origin = spark.conf.get("spark.sql.ansi.enabled")
->>> spark.conf.set("spark.sql.ansi.enabled", "true")
-
->>> df = spark.createDataFrame(
-...     [(Decimal("1" * 38),)] * 10,
-...     "x DECIMAL(38,0)"
-... )
->>> df.select(sf.try_avg("x").alias("avg_x")).show()
-+-----+
-|avg_x|
-+-----+
-| NULL|
-+-----+
-
-
->>> from pyspark.sql import functions as sf, types as T
->>> LONG_MAX = 2**63 - 1
->>> data = [
-...     ("bad",  LONG_MAX),
-...     ("bad",  1),
-...     ("ok",   10),
-...     ("ok",   None),
-...     ("ok",   5),
-... ]
->>> df = spark.createDataFrame(
-...     data,
-...     schema=T.StructType([
-...         T.StructField("g", T.StringType(), True),
-...         T.StructField("x", T.LongType(),   True),
-...     ])
-... )
->>> out = (
-...     df.groupBy("g")
-...       .agg(sf.try_avg("x").alias("avg_x"))
-...       .orderBy("g")
-... )
->>> out.show(truncate=False)
-+---+--------------------+
-|g  |avg_x               |
-+---+--------------------+
-|bad|4.611686018427388e18|
-|ok |7.5                 |
-+---+--------------------+
-
->>> out.printSchema()
-root
- |-- g: string (nullable = true)
- |-- avg_x: double (nullable = true)
-
->>> from pyspark.sql import functions as sf, types as T
-
->>> LONG_MAX = 2**63 - 1
->>> data = [
-...     ("bad",  LONG_MAX),
-...     ("bad",  1),
-...     ("ok",   10),
-...     ("ok",   None),
-...     ("ok",   5),
-... ]
->>> df = spark.createDataFrame(
-...     data,
-...     schema=T.StructType([
-...         T.StructField("g", T.StringType(), True),
-...         T.StructField("x", T.LongType(),   True),
-...     ])
-... )
->>> out = (
-...     df.groupBy("g")
-...       .agg(sf.try_avg("x").alias("avg_x"))
-...       .orderBy("g")
-... )
->>> out.show()
-+---+--------------------+
-|  g|               avg_x|
-+---+--------------------+
-|bad|4.611686018427388e18|
-| ok|                 7.5|
-+---+--------------------+
-
->>> out.printSchema()
-root
- |-- g: string (nullable = true)
- |-- avg_x: double (nullable = true)
-
-# Interval
->>> spark.sql("SELECT try_avg(col) FROM VALUES (interval '2147483647 months'), (interval '1 months') AS tab(col)").show()
-+------------+
-|try_avg(col)|
-+------------+
-|        NULL|
-+------------+
-
->>> spark.sql("SELECT try_avg(col) FROM VALUES (interval '7 months'), (interval '1 months') AS tab(col)").show(truncate=False)
-+----------------------------+
-|try_avg(col)                |
-+----------------------------+
-|INTERVAL '0-4' YEAR TO MONTH|
-+----------------------------+
-
->>> spark.sql("SELECT try_avg(col) FROM VALUES (interval '10 months'), null, (interval '5 months') AS tab(col)").show(truncate=False)
-+----------------------------+
-|try_avg(col)                |
-+----------------------------+
-|INTERVAL '0-8' YEAR TO MONTH|
-+----------------------------+
diff --git a/python/pysail/tests/spark/function/aggregate/test_try_sum.txt b/python/pysail/tests/spark/function/aggregate/test_try_sum.txt
deleted file mode 100644
index 6a51bafb3..000000000
--- a/python/pysail/tests/spark/function/aggregate/test_try_sum.txt
+++ /dev/null
@@ -1,138 +0,0 @@
-# Int
->>> from pyspark.sql import functions as sf
->>> from pyspark.sql.types import LongType
->>> df = spark.createDataFrame([(1,), (2,), (3,)], ["x"])
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-|    6|
-+-----+
-
->>> df = spark.createDataFrame([(None,), (2,), (None,)], ["x"])
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-|    2|
-+-----+
-
->>> LONG_MAX = pow(2,63) -1
->>> df = spark.createDataFrame([(LONG_MAX,), (1,)], ["x"]).withColumn("x", sf.col("x").cast(LongType()))
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-| NULL|
-+-----+
-
-# Float
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame([(1.5,), (2.5,), (3.0,)], ["x"])
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-|  7.0|
-+-----+
-
->>> df = spark.createDataFrame([(1e308,), (1e308,)], ["x"])
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+--------+
-|   sum_x|
-+--------+
-|Infinity|
-+--------+
-
->>> df = spark.createDataFrame([(float('nan'),), (1.0,)], ["x"])
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-|  NaN|
-+-----+
-
->>> df = spark.createDataFrame([(float('inf'),), (1.0,)], ["x"])
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+--------+
-|   sum_x|
-+--------+
-|Infinity|
-+--------+
-
-# Decimal
->>> from decimal import Decimal
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame([(Decimal("1.23"),), (Decimal("4.77"),)], "x DECIMAL(10,2)")
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-| 6.00|
-+-----+
->>> df.select(sf.try_sum("x").alias("sum_x")).printSchema()
-root
- |-- sum_x: decimal(20,2) (nullable = true)
-
->>> df = spark.createDataFrame([(Decimal("1.00"),), (None,), (Decimal("2.50"),)], "x DECIMAL(10,2)")
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-| 3.50|
-+-----+
-
->>> df.select(sf.try_sum("x").alias("sum_x")).printSchema()
-root
- |-- sum_x: decimal(20,2) (nullable = true)
-
->>> from pyspark.sql import functions as sf
->>> from decimal import Decimal
->>> df = spark.createDataFrame([(Decimal("90000"),), (Decimal("20000"),)], "x DECIMAL(5,0)")
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+------+
-| sum_x|
-+------+
-|110000|
-+------+
-
->>> df.select(sf.try_sum("x").alias("sum_x")).printSchema()
-root
- |-- sum_x: decimal(15,0) (nullable = true)
-
-# like test builtin
-# spark.conf.set("spark.sql.ansi.enabled", "true")
->>> from pyspark.sql import functions as sf
->>> from decimal import Decimal
->>> origin = spark.conf.get("spark.sql.ansi.enabled")
->>> spark.conf.set("spark.sql.ansi.enabled", "true")
->>> df = spark.createDataFrame([(Decimal("1" * 38),)] * 10, "x DECIMAL(38,0)")
->>> df.select(sf.try_sum("x").alias("sum_x")).show()
-+-----+
-|sum_x|
-+-----+
-| NULL|
-+-----+
-
->>> df.select(sf.try_sum("x").alias("sum_x")).printSchema()
-root
- |-- sum_x: decimal(38,0) (nullable = true)
-
-# GroupBy
->>> from pyspark.sql import functions as sf, types as T
->>> LONG_MAX = 2**63 - 1
->>> data = [("bad",  LONG_MAX),("bad",  1),("ok",   10),("ok",   None),("ok",   5),]
->>> df = spark.createDataFrame(data, schema=T.StructType([T.StructField("g", T.StringType(), True),T.StructField("x", T.LongType(),   True),]))
->>> out = df.groupBy("g").agg(sf.try_sum("x").alias("sum_x")).orderBy("g")
->>> out.show()
-+---+-----+
-|  g|sum_x|
-+---+-----+
-|bad| NULL|
-| ok|   15|
-+---+-----+
-
->>> out.printSchema()
-root
- |-- g: string (nullable = true)
- |-- sum_x: long (nullable = true)
diff --git a/python/pysail/tests/spark/function/features/aggregate.feature b/python/pysail/tests/spark/function/features/aggregate.feature
deleted file mode 100644
index 5789e4201..000000000
--- a/python/pysail/tests/spark/function/features/aggregate.feature
+++ /dev/null
@@ -1,291 +0,0 @@
-@lambda_hof
-@aggregate
-Feature: aggregate higher-order function
-
-  Rule: Array aggregation with lambda functions
-
-    Scenario: aggregate sums integer array with identity finish
-      When query
-        """
-        SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | 6      |
-
-    Scenario: aggregate applies explicit finish lambda
-      When query
-        """
-        SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10) AS result
-        """
-      Then query result
-        | result |
-        | 60     |
-
-    Scenario: aggregate computes average with struct accumulator
-      When query
-        """
-        SELECT aggregate(
-          array(1, 2, 3, 4),
-          named_struct('sum', 0, 'cnt', 0),
-          (acc, x) -> named_struct('sum', acc.sum + x, 'cnt', acc.cnt + 1),
-          acc -> acc.sum / acc.cnt
-        ) AS avg
-        """
-      Then query result
-        | avg |
-        | 2.5 |
-
-    Scenario: aggregate applies finish to initial value for empty array
-      When query
-        """
-        SELECT aggregate(CAST(array() AS ARRAY), 0, (acc, x) -> acc + x, acc -> acc * 10) AS result
-        """
-      Then query result
-        | result |
-        | 0      |
-
-    Scenario: aggregate returns NULL for NULL array
-      When query
-        """
-        SELECT aggregate(CAST(NULL AS ARRAY), 0, (acc, x) -> acc + x, acc -> acc * 10) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: aggregate handles NULL elements through merge lambda
-      When query
-        """
-        SELECT aggregate(array(1, NULL, 3), 0, (acc, x) -> acc + coalesce(x, 0)) AS result
-        """
-      Then query result
-        | result |
-        | 4      |
-
-    Scenario: aggregate can capture outer columns per row
-      When query
-        """
-        SELECT aggregate(arr, base, (acc, x) -> acc + x) AS result
-        FROM VALUES
-          (array(1, 2), 10),
-          (array(3), 20)
-        AS t(arr, base)
-        """
-      Then query result
-        | result |
-        | 13     |
-        | 23     |
-
-    Scenario: aggregate supports struct accumulator and finish conversion
-      When query
-        """
-        SELECT aggregate(
-          array(
-            CAST(20.0 AS DOUBLE),
-            CAST(4.0 AS DOUBLE),
-            CAST(2.0 AS DOUBLE),
-            CAST(6.0 AS DOUBLE),
-            CAST(10.0 AS DOUBLE)
-          ),
-          named_struct('count', 0, 'sum', CAST(0.0 AS DOUBLE)),
-          (acc, x) -> named_struct('count', acc.count + 1, 'sum', acc.sum + x),
-          acc -> acc.sum / acc.count
-        ) AS result
-        """
-      Then query result
-        | result |
-        | 8.4    |
-
-    Scenario: aggregate merge can reference the element without the accumulator
-      When query
-        """
-        SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> x) AS result
-        """
-      Then query result
-        | result |
-        | 3      |
-
-    Scenario: aggregate merge references the element and a captured column
-      When query
-        """
-        SELECT aggregate(arr, 0, (acc, x) -> x + base) AS result
-        FROM VALUES
-          (array(1, 2), 10),
-          (array(3), 20)
-        AS t(arr, base)
-        """
-      Then query result
-        | result |
-        | 12     |
-        | 23     |
-
-    Scenario: reduce is an alias for aggregate
-      When query
-        """
-        SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | 6      |
-
-    Scenario: reduce applies an explicit finish lambda
-      When query
-        """
-        SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10) AS result
-        """
-      Then query result
-        | result |
-        | 60     |
-
-    Scenario: reduce merge can reference the element without the accumulator
-      When query
-        """
-        SELECT reduce(array(1, 2, 3), 0, (acc, x) -> x) AS result
-        """
-      Then query result
-        | result |
-        | 3      |
-
-    Scenario: reduce returns NULL for NULL array
-      When query
-        """
-        SELECT reduce(CAST(NULL AS ARRAY), 0, (acc, x) -> acc + x, acc -> acc * 10) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Float extremes and all-NULL elements propagate through the fold
-
-    Scenario: aggregate propagates positive infinity
-      When query
-        """
-        SELECT aggregate(array(CAST('Infinity' AS DOUBLE), 1.0), CAST(0.0 AS DOUBLE), (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result   |
-        | Infinity |
-
-    Scenario: aggregate propagates negative infinity
-      When query
-        """
-        SELECT aggregate(array(CAST('-Infinity' AS DOUBLE), 1.0), CAST(0.0 AS DOUBLE), (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result    |
-        | -Infinity |
-
-    Scenario: aggregate propagates NaN
-      When query
-        """
-        SELECT aggregate(array(CAST('NaN' AS DOUBLE), 1.0), CAST(0.0 AS DOUBLE), (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: aggregate of positive and negative infinity is NaN
-      When query
-        """
-        SELECT aggregate(array(CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE)), CAST(0.0 AS DOUBLE), (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: aggregate over an all-NULL-element array propagates NULL
-      When query
-        """
-        SELECT aggregate(CAST(array(NULL, NULL) AS ARRAY), 0, (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: aggregate mixing normal, NULL, infinity and NaN propagates NULL
-      When query
-        """
-        SELECT aggregate(array(1.0, CAST(NULL AS DOUBLE), CAST('Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST(0.0 AS DOUBLE), (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: aggregate mixing normal, NULL, infinity and NaN with coalesce yields NaN
-      When query
-        """
-        SELECT aggregate(array(1.0, CAST(NULL AS DOUBLE), CAST('Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST(0.0 AS DOUBLE), (acc, x) -> acc + coalesce(x, 0)) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: aggregate mixing normal, infinity and NaN yields NaN
-      When query
-        """
-        SELECT aggregate(array(1.0, CAST('Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST(0.0 AS DOUBLE), (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: aggregate mixing normal, NULL and infinity with coalesce yields infinity
-      When query
-        """
-        SELECT aggregate(array(1.0, CAST(NULL AS DOUBLE), CAST('Infinity' AS DOUBLE)), CAST(0.0 AS DOUBLE), (acc, x) -> acc + coalesce(x, 0)) AS result
-        """
-      Then query result
-        | result   |
-        | Infinity |
-
-  Rule: ANSI arithmetic inside the merge lambda
-
-    @sail-bug
-    Scenario: integer overflow inside merge errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT aggregate(array(2000000000, 2000000000), 0, (acc, x) -> acc + x) AS result
-        """
-      Then query error .*
-
-    Scenario: integer overflow inside merge wraps under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT aggregate(array(2000000000, 2000000000), 0, (acc, x) -> acc + x) AS result
-        """
-      Then query result
-        | result     |
-        | -294967296 |
-
-  Rule: Lambda arity is validated against Spark
-
-    Scenario: aggregate rejects a merge lambda with fewer than 2 parameters
-      When query
-        """
-        SELECT aggregate(array(1, 2, 3), 0, x -> x) AS result
-        """
-      Then query error (?i)lambda function
-
-    Scenario: aggregate rejects a merge lambda with more than 2 parameters
-      When query
-        """
-        SELECT aggregate(array(1, 2, 3), 0, (a, b, c) -> a) AS result
-        """
-      Then query error (?i)lambda function
-
-    Scenario: aggregate rejects a finish lambda with more than 1 parameter
-      When query
-        """
-        SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x, (a, b) -> a) AS result
-        """
-      Then query error (?i)lambda function
-
-    Scenario: reduce rejects a merge lambda with fewer than 2 parameters
-      When query
-        """
-        SELECT reduce(array(1, 2, 3), 0, acc -> acc + 1) AS result
-        """
-      Then query error (?i)lambda function
diff --git a/python/pysail/tests/spark/function/features/bitmap_construct_agg.feature b/python/pysail/tests/spark/function/features/aggregate/bitmap_construct_agg.feature
similarity index 57%
rename from python/pysail/tests/spark/function/features/bitmap_construct_agg.feature
rename to python/pysail/tests/spark/function/features/aggregate/bitmap_construct_agg.feature
index db87644c7..2972867b8 100644
--- a/python/pysail/tests/spark/function/features/bitmap_construct_agg.feature
+++ b/python/pysail/tests/spark/function/features/aggregate/bitmap_construct_agg.feature
@@ -2,35 +2,21 @@ Feature: bitmap_construct_agg builds a bitmap from bit positions
 
   Rule: bitmap_construct_agg sets bits from bitmap_bit_position
 
-    Scenario: bitmap_construct_agg with identical input values
+    Scenario Outline: bitmap_construct_agg with 
       When query
         """
         SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) AS result
-        FROM VALUES (1), (1), (1) AS tab(col)
+        FROM VALUES  AS tab(col)
         """
       Then query result
-        | result |
-        | 010000 |
+        | result   |
+        |  |
 
-    Scenario: bitmap_construct_agg with distinct input values
-      When query
-        """
-        SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) AS result
-        FROM VALUES (1), (2), (3) AS tab(col)
-        """
-      Then query result
-        | result |
-        | 070000 |
-
-    Scenario: bitmap_construct_agg with larger bit positions
-      When query
-        """
-        SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) AS result
-        FROM VALUES (1), (9), (17) AS tab(col)
-        """
-      Then query result
-        | result |
-        | 010101 |
+      Examples:
+        | case                   | values         | result |
+        | identical input values | (1), (1), (1)  | 010000 |
+        | distinct input values  | (1), (2), (3)  | 070000 |
+        | larger bit positions   | (1), (9), (17) | 010101 |
 
   Rule: bitmap_construct_agg returns a binary value
 
@@ -46,25 +32,21 @@ Feature: bitmap_construct_agg builds a bitmap from bit positions
 
   Rule: bitmap_construct_agg handles null values
 
-    Scenario: bitmap_construct_agg ignores null input values
+    Scenario Outline: bitmap_construct_agg 
       When query
         """
         SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) AS result
-        FROM VALUES (1), (CAST(NULL AS BIGINT)), (3) AS tab(col)
+        FROM VALUES  AS tab(col)
         """
       Then query result
-        | result |
-        | 050000 |
+        | result   |
+        |  |
 
-    Scenario: bitmap_construct_agg with all null inputs
-      When query
-        """
-        SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) AS result
-        FROM VALUES (CAST(NULL AS BIGINT)), (CAST(NULL AS BIGINT)) AS tab(col)
-        """
-      Then query result
-        | result |
-        | 000000 |
+      Examples:
+        | case                      | values                                         | result |
+        | ignores null input values | (1), (CAST(NULL AS BIGINT)), (3)               | 050000 |
+        | with all null inputs      | (CAST(NULL AS BIGINT)), (CAST(NULL AS BIGINT)) | 000000 |
+        | supports negative values  | (-1), (-2), (-3)                               | 0E0000 |
 
     Scenario: bitmap_construct_agg on empty input returns an empty bitmap
       When query
@@ -76,16 +58,6 @@ Feature: bitmap_construct_agg builds a bitmap from bit positions
         | result |
         | 000000 |
 
-    Scenario: bitmap_construct_agg supports negative input values
-      When query
-        """
-        SELECT substring(hex(bitmap_construct_agg(bitmap_bit_position(col))), 0, 6) AS result
-        FROM VALUES (-1), (-2), (-3) AS tab(col)
-        """
-      Then query result
-        | result |
-        | 0E0000 |
-
   Rule: bitmap_count can count bits in bitmap_construct_agg output
 
     Scenario: bitmap_count of bitmap_construct_agg output
diff --git a/python/pysail/tests/spark/function/features/bitmap_or_agg.feature b/python/pysail/tests/spark/function/features/aggregate/bitmap_or_agg.feature
similarity index 87%
rename from python/pysail/tests/spark/function/features/bitmap_or_agg.feature
rename to python/pysail/tests/spark/function/features/aggregate/bitmap_or_agg.feature
index c8e93864e..a3b960dfe 100644
--- a/python/pysail/tests/spark/function/features/bitmap_or_agg.feature
+++ b/python/pysail/tests/spark/function/features/aggregate/bitmap_or_agg.feature
@@ -48,25 +48,20 @@ Feature: bitmap_or_agg returns the bitwise OR of all input bitmaps
 
   Rule: bitmap_or_agg handles null values
 
-    Scenario: bitmap_or_agg ignores null input values
+    Scenario Outline: bitmap_or_agg 
       When query
         """
         SELECT substring(hex(bitmap_or_agg(col)), 0, 6) AS result
-        FROM VALUES (X'10'), (CAST(NULL AS BINARY)), (X'40') AS tab(col)
+        FROM VALUES  AS tab(col)
         """
       Then query result
-        | result |
-        | 500000 |
+        | result   |
+        |  |
 
-    Scenario: bitmap_or_agg with all null inputs
-      When query
-        """
-        SELECT substring(hex(bitmap_or_agg(col)), 0, 6) AS result
-        FROM VALUES (CAST(NULL AS BINARY)), (CAST(NULL AS BINARY)) AS tab(col)
-        """
-      Then query result
-        | result |
-        | 000000 |
+      Examples:
+        | case                      | values                                         | result |
+        | ignores null input values | (X'10'), (CAST(NULL AS BINARY)), (X'40')       | 500000 |
+        | with all null inputs      | (CAST(NULL AS BINARY)), (CAST(NULL AS BINARY)) | 000000 |
 
     Scenario: bitmap_or_agg on empty input returns an empty bitmap
       When query
diff --git a/python/pysail/tests/spark/function/features/aggregate/collect_list.feature b/python/pysail/tests/spark/function/features/aggregate/collect_list.feature
new file mode 100644
index 000000000..065d02dfe
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/collect_list.feature
@@ -0,0 +1,185 @@
+Feature: collect_list / collect_set
+
+  collect_list gathers all values of a group into an array (order and duplicates preserved);
+  collect_set gathers the distinct values. Both ignore NULLs and return an empty array `[]`
+  for a group with no (non-NULL) values.
+
+  Sail maps both onto DataFusion's array_agg (sail-plan `aggregate.rs`: `array_agg_compacted`
+  for collect_list, `collect_set` for collect_set). DataFusion's array_agg returns NULL over
+  zero rows, so the empty case is coalesced to a typed empty array to match Spark.
+
+  Rule: collect_list collects values of any type, preserving order and duplicates
+
+    Scenario Outline: collect_list over 
+      When query
+        """
+        SELECT collect_list(v) AS r FROM VALUES  AS t(v)
+        """
+      Then query result
+        | r   |
+        |  |
+
+      Examples:
+        | case                                     | values                                                       | r                        |
+        | integers keep input order and duplicates | (1), (2), (1)                                                | [1, 2, 1]                |
+        | strings                                  | ('a'), ('b'), ('a')                                          | [a, b, a]                |
+        | doubles                                  | (1.5D), (2.5D)                                               | [1.5, 2.5]               |
+        | decimals keep their scale                | (CAST(1.50 AS DECIMAL(5, 2))), (CAST(2.00 AS DECIMAL(5, 2))) | [1.50, 2.00]             |
+        | booleans                                 | (true), (false), (true)                                      | [true, false, true]      |
+        | dates                                    | (DATE'2024-01-01'), (DATE'2024-02-02')                       | [2024-01-01, 2024-02-02] |
+        | array elements                           | (array(1, 2)), (array(3))                                    | [[1, 2], [3]]            |
+        | struct elements                          | (named_struct('a', 1)), (named_struct('a', 2))               | [{1}, {2}]               |
+
+    # collect_list accepts map elements (unlike collect_set, which requires orderable elements).
+    Scenario: map elements
+      When query
+        """
+        SELECT collect_list(v) AS r FROM VALUES (map('a', 1)) AS t(v)
+        """
+      Then query result
+        | r          |
+        | [{a -> 1}] |
+
+  Rule: NULLs are ignored
+
+    Scenario: collect_list drops NULLs and keeps the non-NULL values
+      When query
+        """
+        SELECT collect_list(v) AS r FROM VALUES
+          (1), (CAST(NULL AS INT)), (2), (CAST(NULL AS INT))
+        AS t(v)
+        """
+      Then query result
+        | r      |
+        | [1, 2] |
+
+  Rule: An empty or all-NULL group collects to an empty array, not NULL
+
+    Scenario Outline:  over an all-NULL group
+      When query
+        """
+        SELECT (v) AS r FROM VALUES
+          (CAST(NULL AS INT)), (CAST(NULL AS INT))
+        AS t(v)
+        """
+      Then query result
+        | r  |
+        | [] |
+
+      Examples:
+        | fn           |
+        | collect_list |
+        | collect_set  |
+
+    Scenario: collect_list whose FILTER removes every row
+      When query
+        """
+        SELECT collect_list(v) FILTER (WHERE v > 100) AS r FROM VALUES (1), (2) AS t(v)
+        """
+      Then query result
+        | r  |
+        | [] |
+
+    Scenario: an all-NULL group is an empty array while populated groups keep their values
+      When query
+        """
+        SELECT g, collect_list(v) AS r FROM VALUES
+          ('a', 1),
+          ('a', CAST(NULL AS INT)),
+          ('b', CAST(NULL AS INT))
+        AS t(g, v) GROUP BY g
+        """
+      Then query result
+        | g | r   |
+        | a | [1] |
+        | b | []  |
+
+    # The same empty-collection case as it surfaces through PIVOT: the absent (2013, dotNET)
+    # cell collects to [].
+    Scenario: collect_set in a pivot fills an absent combination with an empty array
+      When query
+        """
+        SELECT * FROM (
+          SELECT year, course, earnings FROM VALUES
+            (2012, 'dotNET', 10),
+            (2012, 'Java', 2),
+            (2013, 'Java', 3)
+          AS courseSales(year, course, earnings)
+        ) PIVOT (
+          collect_set(earnings) FOR (course) IN ('dotNET', 'Java')
+        )
+        """
+      Then query result
+        | year | dotNET | Java |
+        | 2012 | [10]   | [2]  |
+        | 2013 | []     | [3]  |
+
+  Rule: collect_set returns the distinct values, ignoring NULLs
+
+    Scenario Outline: Collect_set returns the distinct values, ignoring NULLs: 
+      When query
+        """
+        SELECT sort_array() AS r FROM VALUES  AS t(v)
+        """
+      Then query result
+        | r   |
+        |  |
+
+      Examples:
+        | case                                          | expr                     | values                        | r         |
+        | collect_set removes duplicates                | collect_set(v)           | (1), (2), (1), (3)            | [1, 2, 3] |
+        | collect_set drops NULLs                       | collect_set(v)           | (1), (CAST(NULL AS INT)), (1) | [1]       |
+        | collect_list with DISTINCT removes duplicates | collect_list(DISTINCT v) | (1), (2), (1)                 | [1, 2]    |
+
+  Rule: collect_set float equality diverges from Spark
+
+    # Spark's collect_set never treats two NaNs as equal, so duplicate NaNs are all kept.
+    # Sail (DataFusion's distinct array_agg) deduplicates NaN, dropping the repeat -> divergence.
+    @sail-bug
+    Scenario: collect_set keeps duplicate NaN values
+      When query
+        """
+        SELECT sort_array(collect_set(v)) AS r FROM VALUES
+          (CAST('NaN' AS DOUBLE)), (CAST('NaN' AS DOUBLE)), (1.0D)
+        AS t(v)
+        """
+      Then query result
+        | r               |
+        | [1.0, NaN, NaN] |
+
+    # Spark's collect_set treats -0.0 and 0.0 as equal and keeps a single 0.0. Sail keeps both
+    # -> divergence.
+    @sail-bug
+    Scenario: collect_set deduplicates -0.0 and 0.0 to a single 0.0
+      When query
+        """
+        SELECT sort_array(collect_set(v)) AS r FROM VALUES
+          (CAST('0.0' AS DOUBLE)), (CAST('-0.0' AS DOUBLE))
+        AS t(v)
+        """
+      Then query result
+        | r     |
+        | [0.0] |
+
+  Rule: collect_set requires orderable element types
+
+    # Spark rejects collect_set over MAP (unorderable) at analysis time. Sail accepts it and
+    # returns a result -> divergence (Sail is too lenient).
+    @sail-bug
+    Scenario: collect_set over a map raises an analysis error
+      When query
+        """
+        SELECT collect_set(v) AS r FROM VALUES (map('a', 1)) AS t(v)
+        """
+      Then query error .*
+
+  Rule: FILTER restricts the collected rows
+
+    Scenario: collect_list with FILTER WHERE collects only matching rows
+      When query
+        """
+        SELECT collect_list(v) FILTER (WHERE v > 1) AS r FROM VALUES (1), (2), (3) AS t(v)
+        """
+      Then query result
+        | r      |
+        | [2, 3] |
diff --git a/python/pysail/tests/spark/function/features/aggregate/count_distinct_star.feature b/python/pysail/tests/spark/function/features/aggregate/count_distinct_star.feature
new file mode 100644
index 000000000..c1358faf9
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/count_distinct_star.feature
@@ -0,0 +1,50 @@
+Feature: COUNT(DISTINCT *) function
+
+  Rule: COUNT(DISTINCT *) counts distinct rows
+
+    # The expected header is the column name Spark derives for `COUNT(DISTINCT *)`,
+    # so it is spelled out from the same  slot and stays asserted.
+    Scenario Outline: count distinct star counts distinct rows 
+      When query
+        """
+        SELECT COUNT(DISTINCT *) FROM VALUES  AS t()
+        """
+      Then query result
+        | count(DISTINCT ) |
+        |                     |
+
+      Examples:
+        | case                    | values                                      | cols             | n |
+        | with duplicates         | (1, 1.0, 'a'), (2, 2.0, 'b'), (1, 1.0, 'a') | a, b, c          | 2 |
+        | all same                | (1, 1), (1, 1), (1, 1)                      | a, b             | 1 |
+        | all different           | (1, 1), (2, 2), (3, 3)                      | a, b             | 3 |
+        | single column           | (1), (2), (1)                               | a                | 2 |
+        | mixed case column names | (1, 2), (1, 2), (3, 4)                      | MyCol, UPPER_COL | 2 |
+
+    Scenario: count distinct star with group by
+      When query
+        """
+        SELECT g, COUNT(DISTINCT *) FROM VALUES ('x', 1), ('x', 1), ('x', 2), ('y', 1) AS t(g, v) GROUP BY g ORDER BY g
+        """
+      Then query result ordered
+        | g | count(DISTINCT g, v) |
+        | x | 2                    |
+        | y | 1                    |
+
+  Rule: COUNT(DISTINCT *) with NULLs
+
+    Scenario Outline: count distinct star with NULLs 
+      When query
+        """
+        SELECT COUNT(DISTINCT *) FROM VALUES  AS t(a, b)
+        """
+      Then query result
+        | count(DISTINCT a, b) |
+        |                   |
+
+      Examples:
+        | case                                | values                                                | n |
+        | skips rows where any column is null | (1, 'a'), (NULL, 'b'), (1, 'a')                       | 1 |
+        | all nulls returns zero              | (NULL, NULL), (NULL, NULL)                            | 0 |
+        | with some null columns              | (1, NULL), (2, NULL), (1, NULL)                       | 0 |
+        | mixed nulls and values              | (1, 'a'), (1, NULL), (2, 'b'), (2, 'b'), (NULL, NULL) | 2 |
diff --git a/python/pysail/tests/spark/function/features/aggregate/count_if.feature b/python/pysail/tests/spark/function/features/aggregate/count_if.feature
new file mode 100644
index 000000000..d9fdaeb72
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/count_if.feature
@@ -0,0 +1,44 @@
+Feature: count_if function
+
+  Rule: count_if as window function
+
+    Scenario: count_if over window
+      When query
+        """
+        SELECT value,
+               count_if(value > 2) OVER (ORDER BY value ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS result
+        FROM VALUES (1), (2), (3), (4), (5) AS t(value)
+        ORDER BY value
+        """
+      Then query result ordered
+        | value | result |
+        | 1     | 0      |
+        | 2     | 0      |
+        | 3     | 1      |
+        | 4     | 2      |
+        | 5     | 3      |
+
+    Scenario: count_if over window with RESPECT NULLS
+      When query
+        """
+        SELECT id,
+               flag,
+               count_if(flag) RESPECT NULLS OVER (
+                 ORDER BY id ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
+               ) AS result
+        FROM VALUES
+          (1, CAST(NULL AS BOOLEAN)),
+          (2, true),
+          (3, false),
+          (4, CAST(NULL AS BOOLEAN)),
+          (5, true)
+          AS t(id, flag)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | flag  | result |
+        | 1  | NULL  | 0      |
+        | 2  | true  | 1      |
+        | 3  | false | 1      |
+        | 4  | NULL  | 1      |
+        | 5  | true  | 1      |
diff --git a/python/pysail/tests/spark/function/features/aggregate/first_last.feature b/python/pysail/tests/spark/function/features/aggregate/first_last.feature
new file mode 100644
index 000000000..d10bbc70d
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/first_last.feature
@@ -0,0 +1,116 @@
+Feature: first / last / any_value inherit ordering from an adjacent ORDER BY
+
+  # The row STORAGE order in every VALUES list below is deliberately scrambled so
+  # that it does NOT match k-ascending nor k-descending order. Therefore a passing
+  # result can ONLY come from the inner ORDER BY driving the aggregate, never from
+  # coincidental scan order. All expected values are verified against Spark JVM 4.x
+  # (rule: first = value of the min-key row, last = value of the max-key row).
+  #
+  #   storage: v = 10, 20, 30, 40   with   k = 'b', 'a', 'd', 'c'
+  #   k ASC:  a(20) b(10) c(40) d(30)  -> first=20  last=30
+  #   k DESC: d(30) c(40) b(10) a(20)  -> first=30  last=20
+
+  Rule: An ORDER BY directly below the aggregate makes first/last deterministic
+
+    Scenario Outline:  respects the  inner ORDER BY
+      When query
+        """
+        SELECT (v) AS result
+        FROM (SELECT * FROM VALUES (10, 'b'), (20, 'a'), (30, 'd'), (40, 'c') AS t(v, k) ORDER BY )
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | fn        | order      | ord    | result |
+        | first     | ascending  | k      | 20     |
+        | last      | ascending  | k      | 30     |
+        | first     | descending | k DESC | 30     |
+        | last      | descending | k DESC | 20     |
+        | any_value | ascending  | k      | 20     |
+
+  Rule: An aliased derived table behaves identically to an unaliased one
+
+    Scenario Outline:  respects the inner ORDER BY through 
+      When query
+        """
+        SELECT (v) AS result
+        FROM (SELECT * FROM VALUES (10, 'b'), (20, 'a'), (30, 'd'), (40, 'c') AS t(v, k) ORDER BY k) AS 
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | fn    | alias_desc                            | alias     | result |
+        | first | an aliased derived table              | sub       | 20     |
+        | last  | an aliased derived table              | sub       | 30     |
+        | first | an aliased derived table with columns | sub(v, k) | 20     |
+
+  Rule: The full multi-key ordering drives the result, not just the leading key
+
+    # storage v = 30, 10, 20  with (k1,k2) = (2,'a'), (1,'b'), (1,'a')
+    # sorted by k1 ASC, k2 ASC: (1,'a')->20, (1,'b')->10, (2,'a')->30  -> first=20 last=30
+    Scenario Outline:  honors the secondary sort key
+      When query
+        """
+        SELECT (v) AS result
+        FROM (
+          SELECT * FROM VALUES (30, 2, 'a'), (10, 1, 'b'), (20, 1, 'a') AS t(v, k1, k2)
+          ORDER BY k1 ASC, k2 ASC
+        )
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | fn    | result |
+        | first | 20     |
+        | last  | 30     |
+
+  Rule: Ordering is applied per group
+
+    # storage v = 10, 20, 30, 40  with k = 'b','a','d','c'  g = 0,0,1,1
+    # g=0 sorted by k: a(20) b(10) -> first=20 last=10
+    # g=1 sorted by k: c(40) d(30) -> first=40 last=30
+    Scenario Outline: grouped  respects the inner ORDER BY within each group
+      When query
+        """
+        SELECT g, (v) AS result
+        FROM (
+          SELECT * FROM VALUES (10, 'b', 0), (20, 'a', 0), (30, 'd', 1), (40, 'c', 1) AS t(v, k, g)
+          ORDER BY k
+        )
+        GROUP BY g
+        ORDER BY g
+        """
+      Then query result
+        | g | result |
+        | 0 |    |
+        | 1 |    |
+
+      Examples:
+        | fn    | g0 | g1 |
+        | first | 20 | 40 |
+        | last  | 10 | 30 |
+
+  Rule: Null treatment is honored alongside the inherited ordering
+
+    # storage v = 30, NULL, 10  with k = 'c','a','b'
+    # k ASC: a(NULL) b(10) c(30) -> respect-nulls first=NULL, ignore-nulls first=10
+    Scenario Outline: first  a leading NULL when nulls are 
+      When query
+        """
+        SELECT first() AS result
+        FROM (SELECT * FROM VALUES (30, 'c'), (CAST(NULL AS INT), 'a'), (10, 'b') AS t(v, k) ORDER BY k)
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case  | treatment | args    | result |
+        | keeps | respected | v       | NULL   |
+        | skips | ignored   | v, true | 10     |
diff --git a/python/pysail/tests/spark/function/features/aggregate/grouping_id.feature b/python/pysail/tests/spark/function/features/aggregate/grouping_id.feature
new file mode 100644
index 000000000..bc2f778e1
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/grouping_id.feature
@@ -0,0 +1,90 @@
+Feature: grouping_id returns Spark-compatible grouping-set bit masks
+
+  Rule: grouping_id over grouping analytics
+
+    Scenario: grouping_id and grouping over cube
+      When query
+        """
+        SELECT
+          name,
+          height,
+          grouping(name) AS g_name,
+          grouping(height) AS g_height,
+          grouping_id() AS gid,
+          sum(age) AS total
+        FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height)
+        GROUP BY cube(name, height)
+        ORDER BY gid, name, height
+        """
+      Then query result ordered
+        | name  | height | g_name | g_height | gid | total |
+        | Alice | 165    | 0      | 0        | 0   | 2     |
+        | Bob   | 180    | 0      | 0        | 0   | 5     |
+        | Alice | NULL   | 0      | 1        | 1   | 2     |
+        | Bob   | NULL   | 0      | 1        | 1   | 5     |
+        | NULL  | 165    | 1      | 0        | 2   | 2     |
+        | NULL  | 180    | 1      | 0        | 2   | 5     |
+        | NULL  | NULL   | 1      | 1        | 3   | 7     |
+
+    Scenario: grouping_id accepts the complete grouping column list
+      When query
+        """
+        SELECT name, height, grouping_id(name, height) AS gid, sum(age) AS total
+        FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height)
+        GROUP BY cube(name, height)
+        ORDER BY gid, name, height
+        """
+      Then query result ordered
+        | name  | height | gid | total |
+        | Alice | 165    | 0   | 2     |
+        | Bob   | 180    | 0   | 5     |
+        | Alice | NULL   | 1   | 2     |
+        | Bob   | NULL   | 1   | 5     |
+        | NULL  | 165    | 2   | 2     |
+        | NULL  | 180    | 2   | 5     |
+        | NULL  | NULL   | 3   | 7     |
+
+    Scenario: grouping_id returns long
+      When query
+        """
+        SELECT grouping_id() AS gid
+        FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name)
+        GROUP BY cube(name)
+        """
+      Then query schema
+        """
+        root
+         |-- gid: long (nullable = false)
+        """
+
+    Scenario: grouping_id works inside a window partition
+      When query
+        """
+        SELECT
+          name,
+          height,
+          grouping_id() AS gid,
+          sum(age) AS total,
+          rank() OVER (PARTITION BY grouping_id() ORDER BY sum(age)) AS r
+        FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height)
+        GROUP BY cube(name, height)
+        ORDER BY gid, r, name, height
+        """
+      Then query result ordered
+        | name  | height | gid | total | r |
+        | Alice | 165    | 0   | 2     | 1 |
+        | Bob   | 180    | 0   | 5     | 2 |
+        | Alice | NULL   | 1   | 2     | 1 |
+        | Bob   | NULL   | 1   | 5     | 2 |
+        | NULL  | 165    | 2   | 2     | 1 |
+        | NULL  | 180    | 2   | 5     | 2 |
+        | NULL  | NULL   | 3   | 7     | 1 |
+
+    Scenario: grouping_id rejects mismatched columns
+      When query
+        """
+        SELECT grouping_id(age) AS gid
+        FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height)
+        GROUP BY cube(name, height)
+        """
+      Then query error GROUPING_ID_COLUMN_MISMATCH
diff --git a/python/pysail/tests/spark/function/features/aggregate/input_order.feature b/python/pysail/tests/spark/function/features/aggregate/input_order.feature
new file mode 100644
index 000000000..246ee62ca
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/input_order.feature
@@ -0,0 +1,253 @@
+Feature: input_order
+  Aggregate/window functions observe input row order established by an
+  upstream ORDER BY (migrated SQL-expressible subset of test_input_order.txt).
+
+  Rule: Order-dependent results (migrated from test_input_order.txt doctests)
+
+    Scenario: input_order doctest #1 — first(age) per name
+      When query
+        """
+        SELECT name, first(age) AS f FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) GROUP BY name ORDER BY name
+        """
+      Then query result ordered
+        | name  | f    |
+        | Alice | NULL |
+        | Bob   | 5    |
+
+    Scenario: input_order doctest #2 — last(age) per name
+      When query
+        """
+        SELECT name, last(age) AS l FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) GROUP BY name ORDER BY name
+        """
+      Then query result ordered
+        | name  | l |
+        | Alice | 2 |
+        | Bob   | 5 |
+
+    Scenario: input_order doctest #3 — first(v) ordered by a+b
+      When query
+        """
+        SELECT first(v) AS f FROM (SELECT * FROM VALUES (30,2,2),(10,5,-20),(20,1,1) AS t(v,a,b) ORDER BY a+b)
+        """
+      Then query result
+        | f  |
+        | 10 |
+
+    Scenario: input_order doctest #4 — first(v) sorted CTE
+      When query
+        """
+        WITH sorted AS (SELECT * FROM VALUES (10,'b'),(20,'a'),(30,'d'),(40,'c') AS t(v,k) ORDER BY k) SELECT first(v) AS r FROM sorted
+        """
+      Then query result
+        | r  |
+        | 20 |
+
+    Scenario: input_order doctest #5 — collect_list ordered
+      When query
+        """
+        SELECT collect_list(v) AS r FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | r            |
+        | [10, 20, 30] |
+
+    Scenario: input_order doctest #6 — first over partition
+      When query
+        """
+        SELECT name, age, first(age) OVER (PARTITION BY name) AS f FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | f    |
+        | Alice | NULL | NULL |
+        | Alice | 2    | NULL |
+        | Bob   | 5    | 5    |
+
+    Scenario: input_order doctest #7 — last over partition
+      When query
+        """
+        SELECT name, age, last(age) OVER (PARTITION BY name) AS l FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | l |
+        | Alice | NULL | 2 |
+        | Alice | 2    | 2 |
+        | Bob   | 5    | 5 |
+
+    Scenario: input_order doctest #8 — first(v) single group same key
+      When query
+        """
+        SELECT first(v) AS f FROM (SELECT * FROM VALUES (1,'a'),(2,'a'),(3,'a') AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | f |
+        | 1 |
+
+    Scenario: input_order doctest #9 — union then first/last
+      When query
+        """
+        SELECT first(v) AS f, last(v) AS l FROM (SELECT * FROM (VALUES (2,'b'),(1,'a'),(4,'d'),(3,'c')) AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | f | l |
+        | 1 | 4 |
+
+    Scenario: input_order doctest #10 — first over + last over
+      When query
+        """
+        SELECT name, age, first(age) OVER (PARTITION BY name) AS f, last(age) OVER (PARTITION BY name) AS l FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | f    | l |
+        | Alice | NULL | NULL | 2 |
+        | Alice | 2    | NULL | 2 |
+        | Bob   | 5    | 5    | 5 |
+
+    Scenario: input_order doctest #11 — last over + last over sum
+      When query
+        """
+        SELECT name, age, (last(age) OVER (PARTITION BY name) + last(age) OVER (PARTITION BY name)) AS s FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | s  |
+        | Alice | NULL | 4  |
+        | Alice | 2    | 4  |
+        | Bob   | 5    | 10 |
+
+    Scenario: input_order doctest #12 — sum over + last over
+      When query
+        """
+        SELECT name, age, sum(age) OVER (PARTITION BY name) AS sm, last(age) OVER (PARTITION BY name) AS l FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | sm | l |
+        | Alice | NULL | 2  | 2 |
+        | Alice | 2    | 2  | 2 |
+        | Bob   | 5    | 5  | 5 |
+
+    Scenario: input_order doctest #13 — listagg ordered
+      When query
+        """
+        SELECT listagg(k) AS s FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v,k) ORDER BY v)
+        """
+      Then query result
+        | s   |
+        | abc |
+
+    Scenario: input_order doctest #14 — HAVING first
+      When query
+        """
+        SELECT g, first(v) AS f FROM (SELECT * FROM VALUES (30,1),(10,1),(20,2) AS t(v,g) ORDER BY v) GROUP BY g HAVING first(v) > 15
+        """
+      Then query result
+        | g | f  |
+        | 2 | 20 |
+
+    Scenario: input_order doctest #15 — CUBE first
+      When query
+        """
+        SELECT g, first(v) AS f FROM (SELECT * FROM VALUES (30,1),(10,1),(20,2) AS t(v,g) ORDER BY v) GROUP BY CUBE(g) ORDER BY g NULLS FIRST
+        """
+      Then query result ordered
+        | g    | f  |
+        | NULL | 10 |
+        | 1    | 10 |
+        | 2    | 20 |
+
+    Scenario: input_order doctest #16 — nth_value over ordered
+      When query
+        """
+        SELECT name, age, nth_value(age, 2) OVER (PARTITION BY name ORDER BY age) AS n FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | n    |
+        | Alice | NULL | NULL |
+        | Alice | 2    | 2    |
+        | Bob   | 5    | NULL |
+
+    Scenario: input_order doctest #17 — aliased subquery first
+      When query
+        """
+        SELECT first(v) AS f FROM (SELECT v FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v,k) ORDER BY k))
+        """
+      Then query result
+        | f  |
+        | 10 |
+
+    Scenario: input_order doctest #18 — select v then first
+      When query
+        """
+        SELECT first(v) AS f FROM (SELECT v FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v,k) ORDER BY k))
+        """
+      Then query result
+        | f  |
+        | 10 |
+
+    Scenario: input_order doctest #19 — sum/first/last over
+      When query
+        """
+        SELECT name, age, sum(age) OVER (PARTITION BY name) AS sm, first(age) OVER (PARTITION BY name) AS f, last(age) OVER (PARTITION BY name) AS l FROM (SELECT * FROM VALUES ('Alice', 2), ('Bob', 5), ('Alice', CAST(NULL AS INT)) AS t(name, age) ORDER BY age) ORDER BY name, age NULLS FIRST
+        """
+      Then query result ordered
+        | name  | age  | sm | f    | l |
+        | Alice | NULL | 2  | NULL | 2 |
+        | Alice | 2    | 2  | NULL | 2 |
+        | Bob   | 5    | 5  | 5    | 5 |
+
+    Scenario Outline: input_order doctest  — first/last DISTINCT 
+      When query
+        """
+        SELECT first(DISTINCT v) AS f, last(DISTINCT v) AS l FROM (SELECT * FROM VALUES  AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | f   | l   |
+        |  |  |
+
+      Examples:
+        | case | variant  | values                              | f  | l  |
+        | #20  |          | (30,'c'),(10,'a'),(20,'b')          | 10 | 30 |
+        | #21  | with dup | (30,'c'),(20,'b'),(10,'a'),(20,'d') | 10 | 20 |
+
+    Scenario: input_order doctest #22 — count/first/last
+      When query
+        """
+        SELECT count(*) AS n, first(v) AS f, last(v) AS l FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | n | f  | l  |
+        | 3 | 10 | 30 |
+
+    Scenario: input_order doctest #23 — explode then first
+      When query
+        """
+        SELECT first(v) AS f FROM (SELECT v, e FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v,k) ORDER BY k) LATERAL VIEW explode(array(1)) AS e)
+        """
+      Then query result
+        | f  |
+        | 10 |
+
+    Scenario: input_order doctest #24 — avg ordered
+      When query
+        """
+        SELECT avg(v) AS a FROM (SELECT * FROM VALUES (CAST(1e16 AS DOUBLE),'b'),(CAST(1.0 AS DOUBLE),'c'),(CAST(-1e16 AS DOUBLE),'a') AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | a                  |
+        | 0.3333333333333333 |
+
+    Scenario: input_order doctest #25 — sum ordered
+      When query
+        """
+        SELECT sum(v) AS s FROM (SELECT * FROM VALUES (CAST(1e16 AS DOUBLE),'b'),(CAST(1.0 AS DOUBLE),'c'),(CAST(-1e16 AS DOUBLE),'a') AS t(v,k) ORDER BY k)
+        """
+      Then query result
+        | s   |
+        | 1.0 |
+
+    Scenario: input_order doctest #26 — join then first
+      When query
+        """
+        SELECT first(v) AS f FROM (SELECT l.v FROM VALUES (2,'b'),(1,'a') AS l(v,k) JOIN VALUES ('a'),('b') AS r(k) ON l.k = r.k ORDER BY l.k DESC)
+        """
+      Then query result
+        | f |
+        | 2 |
diff --git a/python/pysail/tests/spark/function/features/aggregate/max_by.feature b/python/pysail/tests/spark/function/features/aggregate/max_by.feature
new file mode 100644
index 000000000..9dfdd9c97
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/max_by.feature
@@ -0,0 +1,49 @@
+Feature: max_by function
+
+  Rule: max_by with all NULLs in ordering column
+
+    Scenario: max_by with all NULLs in ordering column
+      When query
+        """
+        SELECT max_by(name, age) AS result
+        FROM VALUES ('Alice', CAST(NULL AS INT)), ('Bob', CAST(NULL AS INT)) AS t(name, age)
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  Rule: max_by as window function
+
+    Scenario: max_by over window
+      When query
+        """
+        SELECT name, age,
+               max_by(name, age) OVER (ORDER BY age ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS result
+        FROM VALUES ('Alice', 30), ('Bob', 50), ('Carol', 40) AS t(name, age)
+        ORDER BY age
+        """
+      Then query result ordered
+        | name  | age | result |
+        | Alice | 30  | Alice  |
+        | Carol | 40  | Carol  |
+        | Bob   | 50  | Bob    |
+
+  Rule: Result values (migrated from test_max_by.txt doctests)
+
+    # All four doctests share the same alltypes/t_base fixture and differ only in
+    # what they select from, so the fixture is written once and  varies.
+    Scenario Outline: max_by doctest  (result)
+      When query
+        """
+        WITH alltypes AS (SELECT CAST(v AS INT) AS int_col, CAST(v % 3 AS BIGINT) AS bigint_col FROM (SELECT explode(sequence(0, 9)) AS v)), t_base AS (SELECT int_col, bigint_col, CASE WHEN int_col IN (0, 9) THEN NULL ELSE int_col END AS by_col, int_col * 10 AS val_col FROM alltypes) SELECT max_by(val_col, by_col) AS result FROM 
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case | source                                                                                                                | result |
+        | #1   | t_base                                                                                                                | 80     |
+        | #2   | (SELECT CASE WHEN val_col = 80 THEN NULL ELSE val_col END AS val_col, by_col FROM t_base)                             | NULL   |
+        | #3   | t_base WHERE int_col <> 8                                                                                             | 70     |
+        | #4   | (SELECT CASE WHEN val_col = 70 THEN NULL ELSE val_col END AS val_col, by_col, int_col FROM t_base) WHERE int_col <> 8 | NULL   |
diff --git a/python/pysail/tests/spark/function/features/aggregate/min_by.feature b/python/pysail/tests/spark/function/features/aggregate/min_by.feature
new file mode 100644
index 000000000..57d2398d1
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/min_by.feature
@@ -0,0 +1,75 @@
+Feature: min_by function
+
+  Rule: min_by with all NULLs in ordering column
+
+    Scenario: min_by with all NULLs in ordering column
+      When query
+        """
+        SELECT min_by(name, age) AS result
+        FROM VALUES ('Alice', CAST(NULL AS INT)), ('Bob', CAST(NULL AS INT)) AS t(name, age)
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  Rule: min_by as window function
+
+    Scenario: min_by over window
+      When query
+        """
+        SELECT name, age,
+               min_by(name, age) OVER (ORDER BY age ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS result
+        FROM VALUES ('Alice', 30), ('Bob', 50), ('Carol', 40) AS t(name, age)
+        ORDER BY age
+        """
+      Then query result ordered
+        | name  | age | result |
+        | Alice | 30  | Alice  |
+        | Carol | 40  | Alice  |
+        | Bob   | 50  | Alice  |
+
+  Rule: Result values (migrated from test_min_by.txt doctests)
+
+    Scenario: min_by doctest #1 (result)
+      When query
+        """
+        WITH alltypes AS (SELECT CAST(v AS INT) AS int_col, CAST(v % 3 AS BIGINT) AS bigint_col FROM (SELECT explode(sequence(0, 9)) AS v)), t_base AS (SELECT int_col, bigint_col, CASE WHEN int_col IN (0, 9) THEN NULL ELSE int_col END AS by_col, int_col * 10 AS val_col FROM alltypes) SELECT * FROM t_base
+        """
+      Then query result
+        | int_col | bigint_col | by_col | val_col |
+        | 0       | 0          | NULL   | 0       |
+        | 1       | 1          | 1      | 10      |
+        | 2       | 2          | 2      | 20      |
+        | 3       | 0          | 3      | 30      |
+        | 4       | 1          | 4      | 40      |
+        | 5       | 2          | 5      | 50      |
+        | 6       | 0          | 6      | 60      |
+        | 7       | 1          | 7      | 70      |
+        | 8       | 2          | 8      | 80      |
+        | 9       | 0          | NULL   | 90      |
+
+    # Doctests #2, #4 and #5 share the alltypes/t_base fixture and the same
+    # min_by(val_col, by_col) call, differing only in what they select from.
+    Scenario Outline: min_by doctest  (result)
+      When query
+        """
+        WITH alltypes AS (SELECT CAST(v AS INT) AS int_col, CAST(v % 3 AS BIGINT) AS bigint_col FROM (SELECT explode(sequence(0, 9)) AS v)), t_base AS (SELECT int_col, bigint_col, CASE WHEN int_col IN (0, 9) THEN NULL ELSE int_col END AS by_col, int_col * 10 AS val_col FROM alltypes) SELECT min_by(val_col, by_col) AS result FROM 
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case | source                                                                                                                | result |
+        | #2   | t_base                                                                                                                | 10     |
+        | #4   | t_base WHERE int_col <> 1                                                                                             | 20     |
+        | #5   | (SELECT CASE WHEN val_col = 20 THEN NULL ELSE val_col END AS val_col, by_col, int_col FROM t_base) WHERE int_col <> 1 | NULL   |
+
+    Scenario: min_by doctest #3 (result)
+      When query
+        """
+        WITH alltypes AS (SELECT CAST(v AS INT) AS int_col, CAST(v % 3 AS BIGINT) AS bigint_col FROM (SELECT explode(sequence(0, 9)) AS v)), t_base AS (SELECT int_col, bigint_col, CASE WHEN int_col IN (0, 9) THEN NULL ELSE int_col END AS by_col, int_col * 10 AS val_col FROM alltypes) SELECT min_by(val_col, CASE WHEN by_col IS NULL THEN by_col ELSE by_col END) AS result FROM (SELECT CASE WHEN val_col = 10 THEN NULL ELSE val_col END AS val_col, by_col FROM t_base)
+        """
+      Then query result
+        | result |
+        | NULL   |
diff --git a/python/pysail/tests/spark/function/features/aggregate/mode.feature b/python/pysail/tests/spark/function/features/aggregate/mode.feature
new file mode 100644
index 000000000..3a838bbce
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/mode.feature
@@ -0,0 +1,35 @@
+Feature: mode
+
+  Rule: Result values (migrated from test_mode.txt doctests)
+
+    Scenario: mode doctest #1 (result)
+      When query
+        """
+        SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col)
+        """
+      Then query result
+        | mode(col) |
+        | 10        |
+
+    Scenario Outline: mode doctest  (result)
+      When query
+        """
+        SELECT mode() WITHIN GROUP (ORDER BY col) FROM VALUES  AS tab(col)
+        """
+      Then query result
+        | mode() WITHIN GROUP (ORDER BY col DESC) |
+        | 10                                      |
+
+      Examples:
+        | case | values                      |
+        | #2   | (0), (10), (10), (20), (20) |
+        | #3   | (0), (10), (10)             |
+
+    Scenario: mode doctest #4 (result)
+      When query
+        """
+        SELECT mode() WITHIN GROUP (ORDER BY col DESC) FROM VALUES (0), (10), (10), (20), (20) AS tab(col)
+        """
+      Then query result
+        | mode() WITHIN GROUP (ORDER BY col) |
+        | 20                                 |
diff --git a/python/pysail/tests/spark/function/features/aggregate/percentile.feature b/python/pysail/tests/spark/function/features/aggregate/percentile.feature
new file mode 100644
index 000000000..695f4c772
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/percentile.feature
@@ -0,0 +1,146 @@
+Feature: percentile() aggregate function computes percentiles
+
+  Rule: Basic percentile calculation
+
+    # The column alias varies per case, so it drives both the query and the
+    # expected header and the auto-derived name stays asserted.
+    Scenario Outline: percentile 
     
+      When query
+        """
+        SELECT percentile(x, 
    ) AS  FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        |   |
+        |  |
+
+      Examples:
+        | p   | case                       | alias   | values                  | result |
+        | 0.3 | with two values            | p30     | (0), (10)               | 3.0    |
+        | 0.5 | with even number of values | median  | (0), (1), (2), (3)      | 1.5    |
+        | 0.5 | with odd number of values  | median  | (0), (1), (2), (3), (4) | 2.0    |
+        | 0.0 | returns minimum            | min_val | (5), (10), (15)         | 5.0    |
+        | 1.0 | returns maximum            | max_val | (5), (10), (15)         | 15.0   |
+
+  Rule: Quartile calculations
+
+    Scenario Outline: percentile 
     ( quartile)
+      When query
+        """
+        SELECT percentile(x, 
    ) AS  FROM (VALUES (0), (1), (2), (3), (4), (5), (6), (7)) AS t(x)
+        """
+      Then query result
+        |   |
+        |  |
+
+      Examples:
+        | p    | case  | alias | result |
+        | 0.25 | first | q1    | 1.75   |
+        | 0.75 | third | q3    | 5.25   |
+
+    Scenario: multiple percentiles
+      When query
+        """
+        SELECT
+          percentile(x, 0.25) AS q1,
+          percentile(x, 0.50) AS q2,
+          percentile(x, 0.75) AS q3
+        FROM (SELECT id AS x FROM range(100)) AS t
+        """
+      Then query result
+        | q1    | q2   | q3    |
+        | 24.75 | 49.5 | 74.25 |
+
+  Rule: NULL handling
+
+    Scenario Outline: percentile 
+      When query
+        """
+        SELECT percentile(x, 0.5) AS median FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | median   |
+        |  |
+
+      Examples:
+        | case                        | values                              | median |
+        | ignores NULLs               | (NULL), (1), (2), (3), (NULL)       | 2.0    |
+        | with all NULLs returns NULL | (CAST(NULL AS INT)), (NULL), (NULL) | NULL   |
+
+    Scenario: percentile on empty dataset returns NULL
+      When query
+        """
+        SELECT percentile(x, 0.5) AS median FROM (SELECT 1 AS x WHERE false) AS t
+        """
+      Then query result
+        | median |
+        | NULL   |
+
+  Rule: Different numeric types
+
+    Scenario Outline: percentile with 
+      When query
+        """
+        SELECT percentile(x, 0.5) AS median FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | median   |
+        |  |
+
+      Examples:
+        | case            | values                                                 | median |
+        | negative values | (-10), (-5), (0), (5), (10)                            | 0.0    |
+        | single value    | (42)                                                   | 42.0   |
+        | float values    | (0.0), (1.0), (2.5), (3.5), (5.0), (6.0), (7.5), (8.5) | 4.25   |
+        | duplicates      | (1), (2), (2), (2), (3)                                | 2.0    |
+
+  Rule: Group by support
+
+    Scenario: percentile with group by
+      When query
+        """
+        SELECT grp, percentile(value, 0.5) AS median
+        FROM (VALUES ('A', 1), ('A', 2), ('A', 3), ('B', 10), ('B', 20), ('B', 30)) AS t(grp, value)
+        GROUP BY grp
+        ORDER BY grp
+        """
+      Then query result ordered
+        | grp | median |
+        | A   | 2.0    |
+        | B   | 20.0   |
+
+  Rule: Comparison with min/max
+
+    Scenario: percentile 0 and 1 match min and max
+      When query
+        """
+        SELECT
+          min(x) AS min_value,
+          percentile(x, 0.0) AS p_0,
+          percentile(x, 0.5) AS p_50,
+          percentile(x, 1.0) AS p_100,
+          max(x) AS max_value
+        FROM (VALUES (0), (1), (2), (3)) AS t(x)
+        """
+      Then query result
+        | min_value | p_0 | p_50 | p_100 | max_value |
+        | 0         | 0.0 | 1.5  | 3.0   | 3         |
+
+  Rule: Array of percentiles
+
+    Scenario: percentile with array of percentile values
+      When query
+        """
+        SELECT percentile(col, array(0.25, 0.5, 0.75)) AS percentiles FROM (VALUES (0), (1), (2), (3), (4)) AS t(col)
+        """
+      Then query result
+        | percentiles     |
+        | [1.0, 2.0, 3.0] |
+
+    Scenario: percentile with full array of percentile values
+      When query
+        """
+        SELECT percentile(x, array(0.0, 0.25, 0.5, 0.75, 1.0)) AS percentiles FROM (VALUES (0), (10)) AS tab(x)
+        """
+      Then query result
+        | percentiles                |
+        | [0.0, 2.5, 5.0, 7.5, 10.0] |
diff --git a/python/pysail/tests/spark/function/features/aggregate/percentile_disc.feature b/python/pysail/tests/spark/function/features/aggregate/percentile_disc.feature
new file mode 100644
index 000000000..7766b73b5
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/percentile_disc.feature
@@ -0,0 +1,380 @@
+Feature: percentile_disc() returns the discrete percentile for a numeric column
+
+  # Regression coverage for `percentile_disc` over numeric inputs (including
+  # DECIMAL) and the array-of-percentiles form. Prior to these fixes the
+  # signature was `OneOf(Exact([, Float64]))`, which rejected calls
+  # of the form `percentile_disc(array(p1, p2, ...))` at type-coercion time
+  # for ALL numeric inputs (not just decimals). The signature is now
+  # `Signature::user_defined` with `coerce_types` mapping every numeric to
+  # `Float64` (matching Spark's `double` return type) and runtime dispatch in
+  # `accumulator()`.
+
+  Rule: DECIMAL(p, s) inputs are accepted and return DOUBLE (Spark-compat)
+
+    Scenario Outline: percentile_disc  
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY CAST(x AS )) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | p   |
+        | 
     |
+
+      Examples:
+        | pct  | case                             | type           | values                                 | p    |
+        | 0.5  | on DECIMAL(10,2)                 | DECIMAL(10,2)  | (1.50), (2.50), (3.50), (4.50), (5.50) | 3.5  |
+        | 0.5  | on high-precision DECIMAL(38,10) | DECIMAL(38,10) | (1.0), (2.0), (3.0), (4.0), (5.0)      | 3.0  |
+        | 0.0  | on DECIMAL returns minimum       | DECIMAL(10,2)  | (5), (10), (15)                        | 5.0  |
+        | 1.0  | on DECIMAL returns maximum       | DECIMAL(10,2)  | (5), (10), (15)                        | 15.0 |
+        | 0.25 | (first quartile) on DECIMAL      | DECIMAL(10,2)  | (10), (20), (30), (40)                 | 10.0 |
+        | 0.75 | (third quartile) on DECIMAL      | DECIMAL(10,2)  | (10), (20), (30), (40)                 | 30.0 |
+
+  Rule: NULL handling matches Spark (NULLs ignored, all-NULL/empty → NULL)
+
+    Scenario Outline: percentile_disc 
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY ) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | p   |
+        | 
     |
+
+      Examples:
+        | case                                      | order                       | values                              | p    |
+        | ignores NULL DECIMAL values               | CAST(x AS DECIMAL(10,2))    | (NULL), (1.0), (2.0), (3.0), (NULL) | 2.0  |
+        | with all-NULL DECIMAL column returns NULL | CAST(NULL AS DECIMAL(10,2)) | (1), (2), (3)                       | NULL |
+        | ignores NULL ORDER BY values (INT)        | x                           | (1), (CAST(NULL AS INT)), (3)       | 1.0  |
+        | on all-NULL INT column returns NULL       | x                           | (CAST(NULL AS INT)), (NULL), (NULL) | NULL |
+
+    Scenario: percentile_disc on empty input returns NULL
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x) AS p
+        FROM (SELECT 1 AS x WHERE false) AS t
+        """
+      Then query result
+        | p    |
+        | NULL |
+
+  Rule: ORDER BY DESC reverses the percentile direction
+
+    # `percentile_disc`'s index `ceil(p * n) - 1` is asymmetric: the naive
+    # `1 - p` inversion that works for `percentile_cont` returns the WRONG
+    # value here. With DESC sorted `[4,3,2,1]`, `percentile_disc(0.25)` is
+    # position 0 = the max value (4), not position 2 = 3.
+
+    Scenario Outline: percentile_disc  DESC 
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY  DESC) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | p   |
+        | 
     |
+
+      Examples:
+        | pct  | case                      | order                    | values                            | p   |
+        | 0.5  | on DECIMAL                | CAST(x AS DECIMAL(10,2)) | (1.0), (2.0), (3.0), (4.0), (5.0) | 3.0 |
+        | 0.25 | selects from the high end | x                        | (1), (2), (3), (4)                | 4.0 |
+        | 0.75 | selects from the low end  | x                        | (1), (2), (3), (4)                | 2.0 |
+        | 0.0  | returns the maximum       | x                        | (1), (2), (3), (4), (5)           | 5.0 |
+        | 1.0  | returns the minimum       | x                        | (1), (2), (3), (4), (5)           | 1.0 |
+
+  Rule: Single value, duplicates and even-count populations
+
+    Scenario Outline: Population: 
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY ) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | p        |
+        |  |
+
+      Examples:
+        | case                                                   | pct | orderby                  | values                                | result |
+        | percentile_disc on a single value                      | 0.5 | x                        | (42)                                  | 42.0   |
+        | percentile_disc on all-duplicate input                 | 0.5 | x                        | (5), (5), (5), (5)                    | 5.0    |
+        | percentile_disc with even row count picks lower middle | 0.5 | x                        | (1), (2), (3), (4)                    | 2.0    |
+        | percentile_disc 0.5 with negative DECIMAL values       | 0.5 | CAST(x AS DECIMAL(10,2)) | (-10.0), (-5.0), (0.0), (5.0), (10.0) | 0.0    |
+
+  Rule: STRING inputs are implicitly cast to DOUBLE (Spark-compat)
+
+    Scenario: percentile_disc on STRING numeric values
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES ('1.5'), ('2.5'), ('3.5'), ('4.5'), ('5.5')) AS t(x)
+        """
+      Then query result
+        | p   |
+        | 3.5 |
+
+    Scenario: percentile_disc on STRING numeric values with array
+      When query
+        """
+        SELECT percentile_disc(array(0.0, 0.5, 1.0)) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES ('1'), ('2'), ('3'), ('4'), ('5')) AS t(x)
+        """
+      Then query result
+        | p               |
+        | [1.0, 3.0, 5.0] |
+
+  Rule: DECIMAL inputs work under GROUP BY
+
+    Scenario: per-group median on DECIMAL
+      When query
+        """
+        SELECT g, percentile_disc(0.5) WITHIN GROUP (ORDER BY CAST(x AS DECIMAL(10,2))) AS p
+        FROM (VALUES ('A', 1.0), ('A', 2.0), ('A', 3.0), ('B', 10.0), ('B', 20.0)) AS t(g, x)
+        GROUP BY g
+        ORDER BY g
+        """
+      Then query result ordered
+        | g | p    |
+        | A | 2.0  |
+        | B | 10.0 |
+
+  Rule: Array-of-percentiles form returns ARRAY
+
+    Scenario Outline: Array of percentiles: 
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY ) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | p        |
+        |  |
+
+      Examples:
+        | case                                                 | pct                         | orderby                  | values                            | result             |
+        | percentile_disc with array of percentiles on DECIMAL | array(0.25, 0.5, 0.75)      | CAST(x AS DECIMAL(10,2)) | (10), (20), (30), (40)            | [10.0, 20.0, 30.0] |
+        | percentile_disc with array of percentiles on INT     | array(0.25, 0.5, 0.75)      | x                        | (1), (2), (3), (4), (5)           | [2.0, 3.0, 4.0]    |
+        | percentile_disc with array on DOUBLE                 | array(0.0, 0.5, 1.0)        | CAST(x AS DOUBLE)        | (1.0), (2.0), (3.0), (4.0), (5.0) | [1.0, 3.0, 5.0]    |
+        | Empty percentile array returns NULL                  | array()                     | x                        | (1), (2), (3), (4), (5)           | NULL               |
+        | NULL element in percentile array is treated as 0.0   | array(CAST(NULL AS DOUBLE)) | x                        | (1), (2), (3), (4), (5)           | [1.0]              |
+    Scenario: percentile_disc with array under GROUP BY
+      When query
+        """
+        SELECT g, percentile_disc(array(0.25, 0.5, 0.75)) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES ('A', 10), ('A', 20), ('A', 30), ('A', 40), ('B', 100), ('B', 200), ('B', 300), ('B', 400)) AS t(g, x)
+        GROUP BY g
+        ORDER BY g
+        """
+      Then query result ordered
+        | g | p                     |
+        | A | [10.0, 20.0, 30.0]    |
+        | B | [100.0, 200.0, 300.0] |
+
+  Rule: Invalid arguments raise an error
+
+    # Error scenarios pin a keyword from the expected validation message
+    # rather than matching `.*`. This keeps the tests asserting that the
+    # FAILURE PATH is the one we intend (out-of-range vs. type mismatch),
+    # not just that "some error" happened.
+
+    Scenario Outline:  is rejected as out of range
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES (1), (2), (3)) AS t(x)
+        """
+      Then query error .*(out of range|VALUE_OUT_OF_RANGE).*
+
+      Examples:
+        | case                                | pct             |
+        | Negative percentile                 | -0.1            |
+        | Percentile greater than 1           | 1.1             |
+        | Out-of-range value inside the array | array(0.5, 1.5) |
+
+    Scenario: DISTINCT with WITHIN GROUP is rejected
+      When query
+        """
+        SELECT percentile_disc(DISTINCT 0.5) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES (1), (2), (3), (2), (1)) AS t(x)
+        """
+      Then query error .*(DISTINCT|distinct).*
+
+    Scenario Outline:  is rejected as non-numeric
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query error .*(numeric|UNEXPECTED_INPUT_TYPE).*
+
+      Examples:
+        | case                               | pct               | values                                   |
+        | BOOLEAN ORDER BY                   | 0.5               | (true), (false)                          |
+        | DATE ORDER BY                      | 0.5               | (DATE '2024-01-01'), (DATE '2024-01-02') |
+        | ARRAY ORDER BY                     | 0.5               | (array(1)), (array(2))                   |
+        | BOOLEAN percentile arg             | true              | (1), (2), (3)                            |
+        | DATE percentile arg                | DATE '2024-01-01' | (1), (2), (3)                            |
+        | Array of strings as percentile arg | array('0.5')      | (1), (2), (3)                            |
+
+  Rule: FLOAT inputs are accepted and return DOUBLE
+
+    Scenario: percentile_disc 0.5 on FLOAT
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY CAST(x AS FLOAT)) AS p
+        FROM (VALUES (1), (2), (3), (4)) AS t(x)
+        """
+      Then query result
+        | p   |
+        | 2.0 |
+
+  Rule: NaN and Infinity order as the extremes (DOUBLE)
+
+    Scenario Outline: NaN and Infinity: 
+      When query
+        """
+        SELECT percentile_disc() WITHIN GROUP (ORDER BY ) AS p
+        FROM (VALUES ) AS t(x)
+        """
+      Then query result
+        | p        |
+        |  |
+
+      Examples:
+        | case                              | pct | orderby | values                                            | result    |
+        | NaN orders as the maximum         | 1.0 | x       | (double('1.0')), (double('2.0')), (double('NaN')) | NaN       |
+        | NaN is not selected below the top | 0.5 | x       | (double('1.0')), (double('2.0')), (double('NaN')) | 2.0       |
+        | positive Infinity is the maximum  | 1.0 | x       | (double('1.0')), (double('Infinity'))             | Infinity  |
+        | negative Infinity is the minimum  | 0.0 | x       | (double('1.0')), (double('-Infinity'))            | -Infinity |
+
+  Rule: FILTER restricts the aggregated population
+
+    Scenario: percentile_disc with FILTER (WHERE)
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x) FILTER (WHERE x > 1) AS p
+        FROM (VALUES (1), (2), (3), (4)) AS t(x)
+        """
+      Then query result
+        | p   |
+        | 3.0 |
+
+  Rule: percentile_disc works as a window function (OVER)
+
+    # @sail-bug: Spark supports percentile_disc as a window function; Sail's window
+    # resolver has no branch for ordered-set (WITHIN GROUP) aggregates, and
+    # percentile_disc is registered as F::unknown in the window registry. Fixing it
+    # needs: (1) a WITHIN-GROUP branch in resolver/expression/window.rs that routes
+    # the WITHIN GROUP ORDER BY into the aggregate's order_bys, and (2) a real
+    # handler in function/window.rs (replacing F::unknown). Deferred — separate work.
+    @sail-bug
+    Scenario: percentile_disc OVER () broadcasts to every row
+      When query
+        """
+        SELECT x, percentile_disc(0.5) WITHIN GROUP (ORDER BY x) OVER () AS p
+        FROM (VALUES (1), (2), (3), (4)) AS t(x)
+        ORDER BY x
+        """
+      Then query result ordered
+        | x | p   |
+        | 1 | 2.0 |
+        | 2 | 2.0 |
+        | 3 | 2.0 |
+        | 4 | 2.0 |
+
+    @sail-bug
+    Scenario: percentile_disc OVER (PARTITION BY)
+      When query
+        """
+        SELECT g, percentile_disc(0.5) WITHIN GROUP (ORDER BY x) OVER (PARTITION BY g) AS p
+        FROM (VALUES ('a', 1), ('a', 3), ('b', 10), ('b', 20)) AS t(g, x)
+        ORDER BY g, x
+        """
+      Then query result ordered
+        | g | p    |
+        | a | 1.0  |
+        | a | 1.0  |
+        | b | 10.0 |
+        | b | 10.0 |
+
+  Rule: ANSI mode affects STRING coercion of the ORDER BY column
+
+    Scenario: non-numeric STRING under ANSI off yields NULL
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES ('abc'), ('def')) AS t(x)
+        """
+      Then query result
+        | p    |
+        | NULL |
+
+    Scenario: non-numeric STRING under ANSI on errors
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES ('abc'), ('def')) AS t(x)
+        """
+      Then query error (?i)cast
+
+  Rule: More invalid arguments are rejected
+
+    Scenario: NULL percentage is rejected
+      When query
+        """
+        SELECT percentile_disc(NULL) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES (1), (2), (3)) AS t(x)
+        """
+      Then query error (?i)null
+
+    Scenario: TIMESTAMP ORDER BY is rejected
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY CAST(x AS TIMESTAMP)) AS p
+        FROM (VALUES (TIMESTAMP '2024-01-01 00:00:00')) AS t(x)
+        """
+      Then query error .*(numeric|UNEXPECTED_INPUT_TYPE|TIMESTAMP).*
+
+    Scenario: multiple ORDER BY expressions are rejected
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x, y) AS p
+        FROM (VALUES (1, 1), (2, 2)) AS t(x, y)
+        """
+      Then query error (?i)within_group|one value
+
+    Scenario: calendar INTERVAL ORDER BY is rejected
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY make_interval(0, 0, 0, x)) AS p
+        FROM (VALUES (1), (2), (3)) AS t(x)
+        """
+      Then query error .*(numeric|ORDERING|INTERVAL).*
+
+  Rule: ANSI day-time interval inputs return INTERVAL (Spark-compat)
+
+    # Year-month intervals raise NOT_IMPLEMENTED in Spark itself, so only day-time
+    # is testable. Stored as i64 internally and reconstructed to the interval type.
+    Scenario: percentile_disc 0.5 on INTERVAL DAY returns an interval
+      When query
+        """
+        SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES (INTERVAL '1' DAY), (INTERVAL '3' DAY), (INTERVAL '5' DAY)) AS t(x)
+        """
+      Then query result
+        | p                                   |
+        | INTERVAL '3 00:00:00' DAY TO SECOND |
+
+    Scenario: percentile_disc with array of percentiles on INTERVAL DAY
+      When query
+        """
+        SELECT percentile_disc(array(0.0, 0.5, 1.0)) WITHIN GROUP (ORDER BY x) AS p
+        FROM (VALUES (INTERVAL '2' DAY), (INTERVAL '5' DAY), (INTERVAL '11' DAY)) AS t(x)
+        """
+      Then query result
+        | p                                                                                                                |
+        | [INTERVAL '2 00:00:00' DAY TO SECOND, INTERVAL '5 00:00:00' DAY TO SECOND, INTERVAL '11 00:00:00' DAY TO SECOND] |
diff --git a/python/pysail/tests/spark/function/features/product.feature b/python/pysail/tests/spark/function/features/aggregate/product.feature
similarity index 85%
rename from python/pysail/tests/spark/function/features/product.feature
rename to python/pysail/tests/spark/function/features/aggregate/product.feature
index 8a17d07ac..baaf913a2 100644
--- a/python/pysail/tests/spark/function/features/product.feature
+++ b/python/pysail/tests/spark/function/features/aggregate/product.feature
@@ -54,25 +54,20 @@ Feature: product returns the multiplicative product of all non-null input values
 
   Rule: product handles null inputs
 
-    Scenario: product skips null values
+    Scenario Outline: product 
       When query
         """
         SELECT product(col) AS p
-        FROM VALUES (2), (CAST(NULL AS INT)), (3), (4) AS tab(col)
+        FROM VALUES  AS tab(col)
         """
       Then query result
-        | p    |
-        | 24.0 |
+        | p   |
+        | 
     |
 
-    Scenario: product over all-null group returns null
-      When query
-        """
-        SELECT product(col) AS p
-        FROM VALUES (CAST(NULL AS INT)), (CAST(NULL AS INT)) AS tab(col)
-        """
-      Then query result
-        | p    |
-        | NULL |
+      Examples:
+        | case                             | values                                   | p    |
+        | skips null values                | (2), (CAST(NULL AS INT)), (3), (4)       | 24.0 |
+        | over all-null group returns null | (CAST(NULL AS INT)), (CAST(NULL AS INT)) | NULL |
 
     Scenario: product over an empty group returns null
       When query
diff --git a/python/pysail/tests/spark/function/features/aggregate/try_avg.feature b/python/pysail/tests/spark/function/features/aggregate/try_avg.feature
new file mode 100644
index 000000000..32c91d0a7
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/try_avg.feature
@@ -0,0 +1,83 @@
+Feature: try_avg
+
+  Rule: Result values (migrated from test_try_avg.txt doctests)
+
+    Scenario Outline: try_avg doctest  (result)
+      When query
+        """
+        SELECT try_avg(x) AS avg_x FROM VALUES  AS t(x)
+        """
+      Then query result
+        | avg_x   |
+        |  |
+
+      Examples:
+        | case | values                                                                                      | avg_x                |
+        | #1   | (1), (2), (3)                                                                               | 2.0                  |
+        | #3   | (CAST(NULL AS INT)), (2), (CAST(NULL AS INT))                                               | 2.0                  |
+        | #4   | (CAST(9223372036854775807 AS BIGINT)), (CAST(1 AS BIGINT))                                  | 4.611686018427388e18 |
+        | #5   | (CAST(1.5 AS DOUBLE)), (CAST(2.5 AS DOUBLE)), (CAST(3.0 AS DOUBLE))                         | 2.3333333333333335   |
+        | #6   | (CAST(1e308 AS DOUBLE)), (CAST(1e308 AS DOUBLE))                                            | Infinity             |
+        | #7   | (CAST('NaN' AS DOUBLE)), (CAST(1.0 AS DOUBLE))                                              | NaN                  |
+        | #8   | (CAST(1.23 AS DECIMAL(10,2))), (CAST(4.77 AS DECIMAL(10,2)))                                | 3.00                 |
+        | #9   | (CAST(1.00 AS DECIMAL(10,2))), (CAST(NULL AS DECIMAL(10,2))), (CAST(2.50 AS DECIMAL(10,2))) | 1.75                 |
+        | #10  | (CAST(90000 AS DECIMAL(5,0))), (CAST(20000 AS DECIMAL(5,0)))                                | 55000                |
+
+    Scenario: try_avg doctest #11 (result)
+      When query
+        """
+        SELECT try_avg(x) AS avg_x FROM VALUES (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))) AS t(x)
+        """
+      Then query result
+        | avg_x |
+        | NULL  |
+
+    Scenario: try_avg doctest #12 (result)
+      When query
+        """
+        SELECT g, try_avg(x) AS avg_x FROM VALUES ('bad', CAST(9223372036854775807 AS BIGINT)), ('bad', CAST(1 AS BIGINT)), ('ok', CAST(10 AS BIGINT)), ('ok', CAST(NULL AS BIGINT)), ('ok', CAST(5 AS BIGINT)) AS t(g, x) GROUP BY g ORDER BY g
+        """
+      Then query result ordered
+        | g   | avg_x                |
+        | bad | 4.611686018427388e18 |
+        | ok  | 7.5                  |
+
+    Scenario Outline: try_avg doctest  over year-month intervals (result)
+      When query
+        """
+        SELECT try_avg(col) AS r FROM VALUES  AS tab(col)
+        """
+      Then query result
+        | r   |
+        |  |
+
+      Examples:
+        | case | values                                                | r                            |
+        | #14  | (interval '2147483647 months'), (interval '1 months') | NULL                         |
+        | #15  | (interval '7 months'), (interval '1 months')          | INTERVAL '0-4' YEAR TO MONTH |
+        | #16  | (interval '10 months'), null, (interval '5 months')   | INTERVAL '0-8' YEAR TO MONTH |
+
+  Rule: Output schema (migrated from test_try_avg.txt printSchema doctests)
+
+    Scenario: try_avg doctest #2 (schema)
+      When query
+        """
+        SELECT try_avg(x) AS avg_x FROM VALUES (1), (2), (3) AS t(x)
+        """
+      Then query schema
+        """
+        root
+         |-- avg_x: double (nullable = true)
+        """
+
+    Scenario: try_avg doctest #13 (schema)
+      When query
+        """
+        SELECT g, try_avg(x) AS avg_x FROM VALUES ('bad', CAST(9223372036854775807 AS BIGINT)), ('bad', CAST(1 AS BIGINT)), ('ok', CAST(10 AS BIGINT)), ('ok', CAST(NULL AS BIGINT)), ('ok', CAST(5 AS BIGINT)) AS t(g, x) GROUP BY g ORDER BY g
+        """
+      Then query schema
+        """
+        root
+         |-- g: string (nullable = true)
+         |-- avg_x: double (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/aggregate/try_sum.feature b/python/pysail/tests/spark/function/features/aggregate/try_sum.feature
new file mode 100644
index 000000000..84f8c2bdd
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/aggregate/try_sum.feature
@@ -0,0 +1,85 @@
+Feature: try_sum
+
+  Rule: Result values (migrated from test_try_sum.txt doctests)
+
+    Scenario Outline: try_sum doctest  (result)
+      When query
+        """
+        SELECT try_sum(x) AS sum_x FROM VALUES  AS t(x)
+        """
+      Then query result
+        | sum_x   |
+        |  |
+
+      Examples:
+        | case | values                                                                                      | sum_x    |
+        | #1   | (1), (2), (3)                                                                               | 6        |
+        | #2   | (CAST(NULL AS INT)), (2), (CAST(NULL AS INT))                                               | 2        |
+        | #3   | (CAST(9223372036854775807 AS BIGINT)), (CAST(1 AS BIGINT))                                  | NULL     |
+        | #4   | (CAST(1.5 AS DOUBLE)), (CAST(2.5 AS DOUBLE)), (CAST(3.0 AS DOUBLE))                         | 7.0      |
+        | #5   | (CAST(1e308 AS DOUBLE)), (CAST(1e308 AS DOUBLE))                                            | Infinity |
+        | #6   | (CAST('NaN' AS DOUBLE)), (CAST(1.0 AS DOUBLE))                                              | NaN      |
+        | #7   | (CAST('Infinity' AS DOUBLE)), (CAST(1.0 AS DOUBLE))                                         | Infinity |
+        | #8   | (CAST(1.23 AS DECIMAL(10,2))), (CAST(4.77 AS DECIMAL(10,2)))                                | 6.00     |
+        | #10  | (CAST(1.00 AS DECIMAL(10,2))), (CAST(NULL AS DECIMAL(10,2))), (CAST(2.50 AS DECIMAL(10,2))) | 3.50     |
+        | #11  | (CAST(90000 AS DECIMAL(5,0))), (CAST(20000 AS DECIMAL(5,0)))                                | 110000   |
+
+    Scenario: try_sum doctest #13 (result)
+      When query
+        """
+        SELECT try_sum(x) AS sum_x FROM VALUES (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))) AS t(x)
+        """
+      Then query result
+        | sum_x |
+        | NULL  |
+
+    Scenario: try_sum doctest #15 (result)
+      When query
+        """
+        SELECT g, try_sum(x) AS sum_x FROM VALUES ('bad', CAST(9223372036854775807 AS BIGINT)), ('bad', CAST(1 AS BIGINT)), ('ok', CAST(10 AS BIGINT)), ('ok', CAST(NULL AS BIGINT)), ('ok', CAST(5 AS BIGINT)) AS t(g, x) GROUP BY g ORDER BY g
+        """
+      Then query result ordered
+        | g   | sum_x |
+        | bad | NULL  |
+        | ok  | 15    |
+
+  Rule: Output schema (migrated from test_try_sum.txt printSchema doctests)
+
+    Scenario Outline: try_sum doctest  (schema)
+      When query
+        """
+        SELECT try_sum(x) AS sum_x FROM VALUES  AS t(x)
+        """
+      Then query schema
+        """
+        root
+         |-- sum_x:  (nullable = true)
+        """
+
+      Examples:
+        | case | values                                                       | type          |
+        | #9   | (CAST(1.23 AS DECIMAL(10,2))), (CAST(4.77 AS DECIMAL(10,2))) | decimal(20,2) |
+        | #12  | (CAST(90000 AS DECIMAL(5,0))), (CAST(20000 AS DECIMAL(5,0))) | decimal(15,0) |
+
+    Scenario: try_sum doctest #14 (schema)
+      When query
+        """
+        SELECT try_sum(x) AS sum_x FROM VALUES (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))), (CAST('11111111111111111111111111111111111111' AS DECIMAL(38,0))) AS t(x)
+        """
+      Then query schema
+        """
+        root
+         |-- sum_x: decimal(38,0) (nullable = true)
+        """
+
+    Scenario: try_sum doctest #16 (schema)
+      When query
+        """
+        SELECT g, try_sum(x) AS sum_x FROM VALUES ('bad', CAST(9223372036854775807 AS BIGINT)), ('bad', CAST(1 AS BIGINT)), ('ok', CAST(10 AS BIGINT)), ('ok', CAST(NULL AS BIGINT)), ('ok', CAST(5 AS BIGINT)) AS t(g, x) GROUP BY g ORDER BY g
+        """
+      Then query schema
+        """
+        root
+         |-- g: string (nullable = true)
+         |-- sum_x: long (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/aggregate.feature b/python/pysail/tests/spark/function/features/array/aggregate.feature
new file mode 100644
index 000000000..42a19a7ec
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/aggregate.feature
@@ -0,0 +1,166 @@
+@function(lambda)
+Feature: aggregate higher-order function
+
+  Rule: Array aggregation with lambda functions
+
+    Scenario Outline: Lambda: 
+      When query
+        """
+        SELECT aggregate() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                              | args                                                                 | result |
+        | aggregate sums integer array with identity finish                 | array(1, 2, 3), 0, (acc, x) -> acc + x                               | 6      |
+        | aggregate applies explicit finish lambda                          | array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10              | 60     |
+        | aggregate applies finish to initial value for empty array         | CAST(array() AS ARRAY), 0, (acc, x) -> acc + x, acc -> acc * 10 | 0      |
+        | aggregate returns NULL for NULL array                             | CAST(NULL AS ARRAY), 0, (acc, x) -> acc + x, acc -> acc * 10    | NULL   |
+        | aggregate handles NULL elements through merge lambda              | array(1, NULL, 3), 0, (acc, x) -> acc + coalesce(x, 0)               | 4      |
+        | aggregate merge can reference the element without the accumulator | array(1, 2, 3), 0, (acc, x) -> x                                     | 3      |
+
+    Scenario Outline: reduce: 
+      When query
+        """
+        SELECT reduce() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                           | args                                                              | result |
+        | reduce is an alias for aggregate                               | array(1, 2, 3), 0, (acc, x) -> acc + x                            | 6      |
+        | reduce applies an explicit finish lambda                       | array(1, 2, 3), 0, (acc, x) -> acc + x, acc -> acc * 10           | 60     |
+        | reduce merge can reference the element without the accumulator | array(1, 2, 3), 0, (acc, x) -> x                                  | 3      |
+        | reduce returns NULL for NULL array                             | CAST(NULL AS ARRAY), 0, (acc, x) -> acc + x, acc -> acc * 10 | NULL   |
+
+    Scenario: aggregate computes average with struct accumulator
+      When query
+        """
+        SELECT aggregate(
+          array(1, 2, 3, 4),
+          named_struct('sum', 0, 'cnt', 0),
+          (acc, x) -> named_struct('sum', acc.sum + x, 'cnt', acc.cnt + 1),
+          acc -> acc.sum / acc.cnt
+        ) AS avg
+        """
+      Then query result
+        | avg |
+        | 2.5 |
+
+    Scenario: aggregate can capture outer columns per row
+      When query
+        """
+        SELECT aggregate(arr, base, (acc, x) -> acc + x) AS result
+        FROM VALUES
+          (array(1, 2), 10),
+          (array(3), 20)
+        AS t(arr, base)
+        """
+      Then query result
+        | result |
+        | 13     |
+        | 23     |
+
+    Scenario: aggregate supports struct accumulator and finish conversion
+      When query
+        """
+        SELECT aggregate(
+          array(
+            CAST(20.0 AS DOUBLE),
+            CAST(4.0 AS DOUBLE),
+            CAST(2.0 AS DOUBLE),
+            CAST(6.0 AS DOUBLE),
+            CAST(10.0 AS DOUBLE)
+          ),
+          named_struct('count', 0, 'sum', CAST(0.0 AS DOUBLE)),
+          (acc, x) -> named_struct('count', acc.count + 1, 'sum', acc.sum + x),
+          acc -> acc.sum / acc.count
+        ) AS result
+        """
+      Then query result
+        | result |
+        | 8.4    |
+
+    Scenario: aggregate merge references the element and a captured column
+      When query
+        """
+        SELECT aggregate(arr, 0, (acc, x) -> x + base) AS result
+        FROM VALUES
+          (array(1, 2), 10),
+          (array(3), 20)
+        AS t(arr, base)
+        """
+      Then query result
+        | result |
+        | 12     |
+        | 23     |
+
+  Rule: Float extremes and all-NULL elements propagate through the fold
+
+    Scenario Outline: aggregate 
+      When query
+        """
+        SELECT aggregate(, , (acc, x) -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                           | arr                                                                                 | init                | merge                | result    |
+        | propagates positive infinity                                   | array(CAST('Infinity' AS DOUBLE), 1.0)                                              | CAST(0.0 AS DOUBLE) | acc + x              | Infinity  |
+        | propagates negative infinity                                   | array(CAST('-Infinity' AS DOUBLE), 1.0)                                             | CAST(0.0 AS DOUBLE) | acc + x              | -Infinity |
+        | propagates NaN                                                 | array(CAST('NaN' AS DOUBLE), 1.0)                                                   | CAST(0.0 AS DOUBLE) | acc + x              | NaN       |
+        | of positive and negative infinity is NaN                       | array(CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE))                      | CAST(0.0 AS DOUBLE) | acc + x              | NaN       |
+        | over an all-NULL-element array propagates NULL                 | CAST(array(NULL, NULL) AS ARRAY)                                               | 0                   | acc + x              | NULL      |
+        | mixing normal, NULL, infinity and NaN propagates NULL          | array(1.0, CAST(NULL AS DOUBLE), CAST('Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE)) | CAST(0.0 AS DOUBLE) | acc + x              | NULL      |
+        | mixing normal, NULL, infinity and NaN with coalesce yields NaN | array(1.0, CAST(NULL AS DOUBLE), CAST('Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE)) | CAST(0.0 AS DOUBLE) | acc + coalesce(x, 0) | NaN       |
+        | mixing normal, infinity and NaN yields NaN                     | array(1.0, CAST('Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE))                       | CAST(0.0 AS DOUBLE) | acc + x              | NaN       |
+        | mixing normal, NULL and infinity with coalesce yields infinity | array(1.0, CAST(NULL AS DOUBLE), CAST('Infinity' AS DOUBLE))                        | CAST(0.0 AS DOUBLE) | acc + coalesce(x, 0) | Infinity  |
+
+  Rule: ANSI arithmetic inside the merge lambda
+
+    @sail-bug
+    Scenario: integer overflow inside merge errors under ANSI on
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT aggregate(array(2000000000, 2000000000), 0, (acc, x) -> acc + x) AS result
+        """
+      Then query error .*
+
+    Scenario: integer overflow inside merge wraps under ANSI off
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT aggregate(array(2000000000, 2000000000), 0, (acc, x) -> acc + x) AS result
+        """
+      Then query result
+        | result     |
+        | -294967296 |
+
+  Rule: Lambda arity is validated against Spark
+
+    Scenario Outline:  rejects 
+      When query
+        """
+        SELECT (array(1, 2, 3), 0, ) AS result
+        """
+      Then query error (?i)lambda function
+
+      Examples:
+        | fn        | case                                        | lambdas                          |
+        | aggregate | a merge lambda with fewer than 2 parameters | x -> x                           |
+        | aggregate | a merge lambda with more than 2 parameters  | (a, b, c) -> a                   |
+        | aggregate | a finish lambda with more than 1 parameter  | (acc, x) -> acc + x, (a, b) -> a |
+
+    Scenario: reduce rejects a merge lambda with fewer than 2 parameters
+      When query
+        """
+        SELECT reduce(array(1, 2, 3), 0, acc -> acc + 1) AS result
+        """
+      Then query error (?i)lambda function
diff --git a/python/pysail/tests/spark/function/features/array/array.feature b/python/pysail/tests/spark/function/features/array/array.feature
new file mode 100644
index 000000000..429126d09
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array.feature
@@ -0,0 +1,40 @@
+Feature: array output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to array yields the schema Spark declares
+      When query
+        """
+        SELECT array(1, 2, 3) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a non-null column input to array yields the schema Spark declares
+      When query
+        """
+        SELECT array(CAST(id AS INT), 2, 3) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a nullable column input to array stays nullable
+      When query
+        """
+        SELECT array(c, 2, 3) AS result FROM VALUES (1), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_append.feature b/python/pysail/tests/spark/function/features/array/array_append.feature
new file mode 100644
index 000000000..657a3fa35
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_append.feature
@@ -0,0 +1,29 @@
+Feature: array_append output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_append yields the schema Spark declares
+      When query
+        """
+        SELECT array_append(array('b', 'd', 'c', 'a'), 'd') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = true)
+        """
+
+    Scenario: a nullable column input to array_append stays nullable
+      When query
+        """
+        SELECT array_append(c, 'd') AS result FROM VALUES (array('b', 'd', 'c', 'a')), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: string (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_coercion.feature b/python/pysail/tests/spark/function/features/array/array_coercion.feature
new file mode 100644
index 000000000..38e4f72cd
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_coercion.feature
@@ -0,0 +1,61 @@
+Feature: array() type coercion with mixed element types
+
+  Spark's non-ANSI semantics coerce mixed string/numeric arrays to string,
+  not to numeric. DataFusion 54 changed `comparison_coercion` to prefer
+  numeric, which would otherwise reject `array('a', 1)` at runtime.
+
+  Rule: Homogeneous arrays preserve their element type
+
+    Scenario Outline: array of 
+      When query
+        """
+        SELECT array() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case     | args          | result          |
+        | integers | 1, 2, 3       | [1, 2, 3]       |
+        | strings  | 'a', 'b', 'c' | [a, b, c]       |
+        | doubles  | 1.0, 2.5, 3.5 | [1.0, 2.5, 3.5] |
+
+  Rule: Mixed numeric types coerce to the widest numeric type
+
+    Scenario: integer and double coerce to double
+      When query
+        """
+        SELECT array(1, 2.5) AS result
+        """
+      Then query result
+        | result     |
+        | [1.0, 2.5] |
+
+  Rule: Mixed string and numeric types coerce to string (Spark non-ANSI)
+
+    Scenario Outline:  coerce to string
+      When query
+        """
+        SELECT array() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                          | args             | result         |
+        | string and integer            | 'a', 1           | [a, 1]         |
+        | string and double             | 'a', 1.5         | [a, 1.5]       |
+        | multiple strings and numerics | 'a', 1, 2.5, 'b' | [a, 1, 2.5, b] |
+
+  Rule: NULL elements are preserved during coercion
+
+    Scenario: string, numeric and NULL coerce to string with NULL preserved
+      When query
+        """
+        SELECT array('a', 1, NULL, 1.0) AS result
+        """
+      Then query result
+        | result            |
+        | [a, 1, NULL, 1.0] |
diff --git a/python/pysail/tests/spark/function/features/array/array_compact.feature b/python/pysail/tests/spark/function/features/array/array_compact.feature
new file mode 100644
index 000000000..5d9385481
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_compact.feature
@@ -0,0 +1,141 @@
+Feature: array_compact() removes null values from an array
+
+  Rule: Basic usage
+
+    Scenario Outline: array_compact 
+      When query
+        """
+        SELECT array_compact() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                         | arr                                         | result    |
+        | removes null values from integer array       | array(1, NULL, 2, NULL, 3)                  | [1, 2, 3] |
+        | removes null values from string array        | array('a', NULL, 'b', NULL, 'c')            | [a, b, c] |
+        | with no null values returns same array       | array(1, 2, 3)                              | [1, 2, 3] |
+        | with all null values returns empty array     | array(CAST(NULL AS INT), CAST(NULL AS INT)) | []        |
+        | with untyped null values returns empty array | array(NULL, NULL)                           | []        |
+        | with null at beginning                       | array(NULL, 1, 2, 3)                        | [1, 2, 3] |
+        | with null at end                             | array(1, 2, 3, NULL)                        | [1, 2, 3] |
+
+  Rule: Empty array handling
+
+    Scenario: array_compact with empty array returns empty array
+      When query
+        """
+        SELECT array_compact(array()) AS result
+        """
+      Then query result
+        | result |
+        | []     |
+
+  Rule: Null input propagation
+
+    Scenario Outline: array_compact with  null input returns null
+      When query
+        """
+        SELECT array_compact() AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case  | arg                      |
+        | an    | NULL                     |
+        | typed | CAST(NULL AS ARRAY) |
+
+  Rule: Multiple rows
+
+    Scenario: array_compact across multiple rows
+      When query
+        """
+        SELECT id, array_compact(arr) AS result
+        FROM VALUES
+          (1, array(1, NULL, 2)),
+          (2, array(NULL, NULL)),
+          (3, array(3, 4, 5)),
+          (4, CAST(NULL AS ARRAY))
+        AS t(id, arr)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | result    |
+        | 1  | [1, 2]    |
+        | 2  | []        |
+        | 3  | [3, 4, 5] |
+        | 4  | NULL      |
+
+  Rule: Various data types
+
+    Scenario Outline: array_compact with  values
+      When query
+        """
+        SELECT array_compact() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case    | arr                         | result        |
+        | double  | array(1.1, NULL, 2.2, NULL) | [1.1, 2.2]    |
+        | boolean | array(true, NULL, false)    | [true, false] |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null array literal yields a non-nullable array
+      When query
+        """
+        SELECT array_compact(array(1, NULL, 2)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null array column yields a non-nullable array
+      When query
+        """
+        SELECT array_compact(array(id)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: long (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT array_compact(c) AS result FROM VALUES (array(1)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: removing NULLs makes the element non-nullable regardless of input
+      When query
+        """
+        SELECT array_compact(c) AS result FROM VALUES (array(1, CAST(NULL AS INT))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_concat.feature b/python/pysail/tests/spark/function/features/array/array_concat.feature
new file mode 100644
index 000000000..562801db7
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_concat.feature
@@ -0,0 +1,56 @@
+@sail-only
+Feature: array_concat() concatenates arrays (Sail extension)
+
+  Note: array_concat is a Sail extension not available in standard Spark.
+  Use concat() for Spark-compatible array concatenation.
+
+  Rule: Basic concatenation
+
+    @sail-only
+    Scenario Outline: array_concat two  arrays
+      When query
+        """
+        SELECT array_concat(, ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case    | left            | right       | result          |
+        | integer | array(1, 2, 3)  | array(4, 5) | [1, 2, 3, 4, 5] |
+        | string  | array('a', 'b') | array('c')  | [a, b, c]       |
+
+  Rule: Empty array handling
+
+    @sail-only
+    Scenario Outline: array_concat 
+      When query
+        """
+        SELECT array_concat(, ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                         | left        | right          | result    |
+        | empty array with typed array | array()     | array(1, 2, 3) | [1, 2, 3] |
+        | typed array with empty array | array(1, 2) | array()        | [1, 2]    |
+
+  Rule: Null propagation
+
+    @sail-only
+    Scenario Outline: array_concat  returns null
+      When query
+        """
+        SELECT array_concat(, ) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case            | left                     | right                    |
+        | array with null | array(1, 2)              | CAST(NULL AS ARRAY) |
+        | null with array | CAST(NULL AS ARRAY) | array(1, 2)              |
diff --git a/python/pysail/tests/spark/function/features/array/array_contains.feature b/python/pysail/tests/spark/function/features/array/array_contains.feature
new file mode 100644
index 000000000..74fa8892c
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_contains.feature
@@ -0,0 +1,27 @@
+Feature: array_contains output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_contains yields the schema Spark declares
+      When query
+        """
+        SELECT array_contains(array(1, 2, 3), 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = false)
+        """
+
+    Scenario: a nullable column input to array_contains stays nullable
+      When query
+        """
+        SELECT array_contains(c, 2) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_distinct.feature b/python/pysail/tests/spark/function/features/array/array_distinct.feature
new file mode 100644
index 000000000..e89452968
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_distinct.feature
@@ -0,0 +1,29 @@
+Feature: array_distinct output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_distinct yields the schema Spark declares
+      When query
+        """
+        SELECT array_distinct(array(1, 2, 3, null, 3)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
+
+    Scenario: a nullable column input to array_distinct stays nullable
+      When query
+        """
+        SELECT array_distinct(c) AS result FROM VALUES (array(1, 2, 3, null, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_except.feature b/python/pysail/tests/spark/function/features/array/array_except.feature
new file mode 100644
index 000000000..0d1725188
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_except.feature
@@ -0,0 +1,29 @@
+Feature: array_except output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_except yields the schema Spark declares
+      When query
+        """
+        SELECT array_except(array(1, 2, 3), array(1, 3, 5)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a nullable column input to array_except stays nullable
+      When query
+        """
+        SELECT array_except(c, array(1, 3, 5)) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array_filter.feature b/python/pysail/tests/spark/function/features/array/array_filter.feature
similarity index 50%
rename from python/pysail/tests/spark/function/features/array_filter.feature
rename to python/pysail/tests/spark/function/features/array/array_filter.feature
index 8c3d0a5a7..cd6b7b2fc 100644
--- a/python/pysail/tests/spark/function/features/array_filter.feature
+++ b/python/pysail/tests/spark/function/features/array/array_filter.feature
@@ -1,141 +1,76 @@
-@lambda_hof
+@function(lambda)
 Feature: array filter with lambda
 
   Rule: Filter array elements using lambda predicates
 
-    Scenario: Filter integers greater than a value
+    Scenario Outline: Comparison predicate: 
       When query
         """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x > 2) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result    |
-        | [3, 4, 5] |
-
-    Scenario: Filter integers less than a value
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x < 3) AS result
-        """
-      Then query result
-        | result |
-        | [1, 2] |
-
-    Scenario: Filter integers greater than or equal to a value
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x >= 3) AS result
-        """
-      Then query result
-        | result    |
-        | [3, 4, 5] |
-
-    Scenario: Filter integers less than or equal to a value
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x <= 2) AS result
-        """
-      Then query result
-        | result |
-        | [1, 2] |
-
-    Scenario: Filter where all elements match
-      When query
-        """
-        SELECT filter(array(10, 20, 30), x -> x > 5) AS result
-        """
-      Then query result
-        | result       |
-        | [10, 20, 30] |
-
-    Scenario: Filter where no elements match
-      When query
-        """
-        SELECT filter(array(1, 2, 3), x -> x > 10) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: Filter with reversed comparison
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> 3 < x) AS result
-        """
-      Then query result
-        | result |
-        | [4, 5] |
+        | result   |
+        |  |
+
+      Examples:
+        | case                                             | arr                  | pred   | result       |
+        | Filter integers greater than a value             | array(1, 2, 3, 4, 5) | x > 2  | [3, 4, 5]    |
+        | Filter integers less than a value                | array(1, 2, 3, 4, 5) | x < 3  | [1, 2]       |
+        | Filter integers greater than or equal to a value | array(1, 2, 3, 4, 5) | x >= 3 | [3, 4, 5]    |
+        | Filter integers less than or equal to a value    | array(1, 2, 3, 4, 5) | x <= 2 | [1, 2]       |
+        | Filter where all elements match                  | array(10, 20, 30)    | x > 5  | [10, 20, 30] |
+        | Filter where no elements match                   | array(1, 2, 3)       | x > 10 | []           |
+        | Filter with reversed comparison                  | array(1, 2, 3, 4, 5) | 3 < x  | [4, 5]       |
 
   Rule: Filter with index argument
 
-    Scenario: Filter using element index - keep elements at even indices
+    Scenario Outline: Index argument: 
       When query
         """
-        SELECT filter(array(10, 20, 30, 40, 50), (x, i) -> i % 2 = 0) AS result
+        SELECT filter(, (x, i) -> ) AS result
         """
       Then query result
-        | result         |
-        | [10, 30, 50]   |
+        | result   |
+        |  |
 
-    Scenario: Filter using element and index - element greater than index
-      When query
-        """
-        SELECT filter(array(0, 5, 1, 10, 2), (x, i) -> x > i) AS result
-        """
-      Then query result
-        | result    |
-        | [5, 10]   |
+      Examples:
+        | case                                                        | arr                       | pred      | result       |
+        | Filter using element index - keep elements at even indices  | array(10, 20, 30, 40, 50) | i % 2 = 0 | [10, 30, 50] |
+        | Filter using element and index - element greater than index | array(0, 5, 1, 10, 2)     | x > i     | [5, 10]      |
 
   Rule: Filter with complex expressions
 
-    Scenario: Filter with AND condition
+    Scenario Outline: Complex expression: 
       When query
         """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x > 1 AND x < 5) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result      |
-        | [2, 3, 4]   |
-
-    Scenario: Filter with OR condition
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x = 1 OR x = 5) AS result
-        """
-      Then query result
-        | result |
-        | [1, 5] |
+        | result   |
+        |  |
 
-    Scenario: Filter with modulo function
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5, 6), x -> x % 2 = 0) AS result
-        """
-      Then query result
-        | result      |
-        | [2, 4, 6]   |
+      Examples:
+        | case                        | arr                     | pred            | result    |
+        | Filter with AND condition   | array(1, 2, 3, 4, 5)    | x > 1 AND x < 5 | [2, 3, 4] |
+        | Filter with OR condition    | array(1, 2, 3, 4, 5)    | x = 1 OR x = 5  | [1, 5]    |
+        | Filter with modulo function | array(1, 2, 3, 4, 5, 6) | x % 2 = 0       | [2, 4, 6] |
 
   Rule: Filter with external column references
 
-    Scenario: Filter using external column as threshold
+    Scenario Outline: External column: 
       When query
         """
-        SELECT filter(arr, x -> x > threshold) AS result
-        FROM (SELECT array(1, 2, 3, 4, 5) AS arr, 2 AS threshold)
+        SELECT filter(arr, x -> ) AS result
+        FROM (SELECT array(1, 2, 3, 4, 5) AS arr, )
         """
       Then query result
-        | result    |
-        | [3, 4, 5] |
+        | result   |
+        |  |
 
-    Scenario: Filter using multiple external columns
-      When query
-        """
-        SELECT filter(arr, x -> x > min_val AND x < max_val) AS result
-        FROM (SELECT array(1, 2, 3, 4, 5) AS arr, 1 AS min_val, 5 AS max_val)
-        """
-      Then query result
-        | result      |
-        | [2, 3, 4]   |
+      Examples:
+        | case                                      | pred                        | cols                       | result    |
+        | Filter using external column as threshold | x > threshold               | 2 AS threshold             | [3, 4, 5] |
+        | Filter using multiple external columns    | x > min_val AND x < max_val | 1 AS min_val, 5 AS max_val | [2, 3, 4] |
 
     Scenario: Filter with varying thresholds per row
       When query
@@ -147,38 +82,26 @@ Feature: array filter with lambda
         AS t(arr, threshold)
         """
       Then query result
-        | result       |
-        | [3, 4, 5]    |
-        | [20, 30]     |
+        | result    |
+        | [3, 4, 5] |
+        | [20, 30]  |
 
   Rule: Filter with null handling
 
-    Scenario: Filter array containing nulls - nulls are excluded by predicate
-      When query
-        """
-        SELECT filter(array(1, NULL, 3, NULL, 5), x -> x > 2) AS result
-        """
-      Then query result
-        | result |
-        | [3, 5] |
-
-    Scenario: Filter with IS NOT NULL predicate
+    Scenario Outline: Null handling: 
       When query
         """
-        SELECT filter(array(1, NULL, 3, NULL, 5), x -> x IS NOT NULL) AS result
+        SELECT filter(array(1, NULL, 3, NULL, 5), x -> ) AS result
         """
       Then query result
-        | result    |
-        | [1, 3, 5] |
+        | result   |
+        |  |
 
-    Scenario: Filter with IS NULL predicate
-      When query
-        """
-        SELECT filter(array(1, NULL, 3, NULL, 5), x -> x IS NULL) AS result
-        """
-      Then query result
-        | result       |
-        | [NULL, NULL] |
+      Examples:
+        | case                                                            | pred          | result       |
+        | Filter array containing nulls - nulls are excluded by predicate | x > 2         | [3, 5]       |
+        | Filter with IS NOT NULL predicate                               | x IS NOT NULL | [1, 3, 5]    |
+        | Filter with IS NULL predicate                                   | x IS NULL     | [NULL, NULL] |
 
   Rule: Filter with different data types
 
@@ -193,70 +116,38 @@ Feature: array filter with lambda
         | result |
         | []     |
 
-    Scenario: Filter string array
+    Scenario Outline: Data type: 
       When query
         """
-        SELECT filter(array('apple', 'banana', 'cherry'), x -> x > 'b') AS result
-        """
-      Then query result
-        | result             |
-        | [banana, cherry]   |
-
-    Scenario: Filter string array with length condition
-      When query
-        """
-        SELECT filter(array('a', 'bb', 'ccc', 'dddd'), x -> length(x) > 2) AS result
-        """
-      Then query result
-        | result      |
-        | [ccc, dddd] |
-
-    Scenario: Filter with negative numbers
-      When query
-        """
-        SELECT filter(array(-5, -2, 0, 3, 7), x -> x >= 0) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
         | result   |
-        | [0, 3, 7] |
+        |  |
 
-    Scenario: Filter double array
-      When query
-        """
-        SELECT filter(array(1.5, 2.7, 3.2, 4.8), x -> x > 2.5) AS result
-        """
-      Then query result
-        | result          |
-        | [2.7, 3.2, 4.8] |
+      Examples:
+        | case                                      | arr                                | pred          | result           |
+        | Filter string array                       | array('apple', 'banana', 'cherry') | x > 'b'       | [banana, cherry] |
+        | Filter string array with length condition | array('a', 'bb', 'ccc', 'dddd')    | length(x) > 2 | [ccc, dddd]      |
+        | Filter with negative numbers              | array(-5, -2, 0, 3, 7)             | x >= 0        | [0, 3, 7]        |
+        | Filter double array                       | array(1.5, 2.7, 3.2, 4.8)          | x > 2.5       | [2.7, 3.2, 4.8]  |
 
   Rule: Filter with equality and other operators
 
-    Scenario: Filter with equality
+    Scenario Outline: Equality operator: 
       When query
         """
-        SELECT filter(array(1, 2, 3, 2, 1), x -> x = 2) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result |
-        | [2, 2] |
+        | result   |
+        |  |
 
-    Scenario: Filter with not equal
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 2, 1), x -> x <> 2) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 3, 1] |
-
-    Scenario: Filter with NOT condition
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> NOT (x > 3)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
+      Examples:
+        | case                      | arr                  | pred        | result    |
+        | Filter with equality      | array(1, 2, 3, 2, 1) | x = 2       | [2, 2]    |
+        | Filter with not equal     | array(1, 2, 3, 2, 1) | x <> 2      | [1, 3, 1] |
+        | Filter with NOT condition | array(1, 2, 3, 4, 5) | NOT (x > 3) | [1, 2, 3] |
 
   Rule: Filter with operators
 
@@ -271,45 +162,36 @@ Feature: array filter with lambda
 
   Rule: Filter with functions in predicate
 
-    Scenario: Filter with function call in predicate
+    Scenario Outline: Function in predicate: 
       When query
         """
-        SELECT filter(array('a', 'bb', 'ccc'), x -> length(x) > 1) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result      |
-        | [bb, ccc]   |
+        | result   |
+        |  |
 
-    Scenario: Filter with arithmetic in predicate
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x * 2 > 5) AS result
-        """
-      Then query result
-        | result    |
-        | [3, 4, 5] |
+      Examples:
+        | case                                   | arr                     | pred          | result    |
+        | Filter with function call in predicate | array('a', 'bb', 'ccc') | length(x) > 1 | [bb, ccc] |
+        | Filter with arithmetic in predicate    | array(1, 2, 3, 4, 5)    | x * 2 > 5     | [3, 4, 5] |
 
   Rule: Filter with index and external columns combined
 
-    Scenario: Filter using both index and external column
+    Scenario Outline: Index and external column: 
       When query
         """
-        SELECT filter(arr, (x, i) -> x > threshold AND i > 0) AS result
+        SELECT filter(arr, (x, i) -> ) AS result
         FROM (SELECT array(1, 5, 2, 8, 3) AS arr, 2 AS threshold)
         """
       Then query result
         | result   |
-        | [5, 8, 3] |
+        |  |
 
-    Scenario: Filter with index less than value and external threshold
-      When query
-        """
-        SELECT filter(arr, (x, i) -> x > threshold AND i < 3) AS result
-        FROM (SELECT array(1, 5, 2, 8, 3) AS arr, 2 AS threshold)
-        """
-      Then query result
-        | result |
-        | [5]    |
+      Examples:
+        | case                                                     | pred                    | result    |
+        | Filter using both index and external column              | x > threshold AND i > 0 | [5, 8, 3] |
+        | Filter with index less than value and external threshold | x > threshold AND i < 3 | [5]       |
 
   Rule: Filter with multiple rows (batch processing)
 
@@ -347,23 +229,19 @@ Feature: array filter with lambda
 
   Rule: Filter with null array input
 
-    Scenario: Filter a null array returns null
+    Scenario Outline: Null array literal: 
       When query
         """
-        SELECT filter(CAST(NULL AS ARRAY), x -> x > 0) AS result
+        SELECT filter(, ) AS result
         """
       Then query result
         | result |
         | NULL   |
 
-    Scenario: Filter null array of strings returns null
-      When query
-        """
-        SELECT filter(CAST(NULL AS ARRAY), x -> x IS NOT NULL) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
+      Examples:
+        | case                                      | arr                         | lambda             |
+        | Filter a null array returns null          | CAST(NULL AS ARRAY)    | x -> x > 0         |
+        | Filter null array of strings returns null | CAST(NULL AS ARRAY) | x -> x IS NOT NULL |
 
     Scenario: Filter multi-row table with null array row
       When query
@@ -393,92 +271,68 @@ Feature: array filter with lambda
 
   Rule: Filter with null elements - predicate returns null treated as false
 
-    Scenario: NULL elements excluded when predicate returns NULL
+    Scenario Outline: Null element: 
       When query
         """
-        SELECT filter(array(1, 2, 3, NULL), x -> x > 1) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result |
-        | [2, 3] |
+        | result   |
+        |  |
 
-    Scenario: All-null array filtered with IS NOT NULL predicate yields empty array
-      When query
-        """
-        SELECT filter(array(NULL, NULL), x -> x IS NOT NULL) AS result
-        """
-      Then query result
-        | result |
-        | []     |
+      Examples:
+        | case                                                                  | arr                  | pred          | result |
+        | NULL elements excluded when predicate returns NULL                    | array(1, 2, 3, NULL) | x > 1         | [2, 3] |
+        | All-null array filtered with IS NOT NULL predicate yields empty array | array(NULL, NULL)    | x IS NOT NULL | []     |
 
   Rule: Filter with boolean arrays
 
-    Scenario: Filter boolean array keeping true values
+    Scenario Outline: Boolean array: 
       When query
         """
-        SELECT filter(array(true, false, true, false), x -> x = true) AS result
+        SELECT filter(array(true, false, true, false), x -> ) AS result
         """
       Then query result
-        | result       |
-        | [true, true] |
+        | result   |
+        |  |
 
-    Scenario: Filter boolean array keeping false values
-      When query
-        """
-        SELECT filter(array(true, false, true, false), x -> x = false) AS result
-        """
-      Then query result
-        | result         |
-        | [false, false] |
+      Examples:
+        | case                                      | pred      | result         |
+        | Filter boolean array keeping true values  | x = true  | [true, true]   |
+        | Filter boolean array keeping false values | x = false | [false, false] |
 
   Rule: Filter with nested arrays
 
-    Scenario: Filter nested array by first element of inner array
+    Scenario Outline: Nested array: 
       When query
         """
-        SELECT filter(array(array(1, 2), array(3, 4), array(5, 6)), x -> x[0] > 2) AS result
+        SELECT filter(array(array(1, 2), array(3, 4), array(5, 6)), x -> ) AS result
         """
       Then query result
-        | result           |
-        | [[3, 4], [5, 6]] |
+        | result   |
+        |  |
 
-    Scenario: Filter nested array - all inner arrays have size greater than 1
-      When query
-        """
-        SELECT filter(array(array(1, 2), array(3, 4), array(5, 6)), x -> size(x) > 1) AS result
-        """
-      Then query result
-        | result                   |
-        | [[1, 2], [3, 4], [5, 6]] |
+      Examples:
+        | case                                                            | pred        | result                   |
+        | Filter nested array by first element of inner array             | x[0] > 2    | [[3, 4], [5, 6]]         |
+        | Filter nested array - all inner arrays have size greater than 1 | size(x) > 1 | [[1, 2], [3, 4], [5, 6]] |
 
   Rule: Filter with constant predicates
 
-    Scenario: Constant true predicate keeps all elements
+    Scenario Outline: Constant predicate: 
       When query
         """
-        SELECT filter(array(1, 2, 3), x -> true) AS result
+        SELECT filter(array(1, 2, 3), x -> ) AS result
         """
       Then query result
-        | result    |
-        | [1, 2, 3] |
-
-    Scenario: Constant false predicate empties the array
-      When query
-        """
-        SELECT filter(array(1, 2, 3), x -> false) AS result
-        """
-      Then query result
-        | result |
-        | []     |
+        | result   |
+        |  |
 
-    Scenario: Constant NULL predicate empties the array
-      When query
-        """
-        SELECT filter(array(1, 2, 3), x -> CAST(NULL AS BOOLEAN)) AS result
-        """
-      Then query result
-        | result |
-        | []     |
+      Examples:
+        | case                                       | pred                  | result    |
+        | Constant true predicate keeps all elements | true                  | [1, 2, 3] |
+        | Constant false predicate empties the array | false                 | []        |
+        | Constant NULL predicate empties the array  | CAST(NULL AS BOOLEAN) | []        |
 
     Scenario: Constant true predicate keeps null array row as NULL
       When query
@@ -639,52 +493,36 @@ Feature: array filter with lambda
 
   Rule: Filter with constant predicate preserves null elements
 
-    Scenario: Constant true predicate keeps null elements
+    Scenario Outline: Constant predicate over null elements: 
       When query
         """
-        SELECT filter(array(1, NULL, 3), x -> true) AS result
+        SELECT filter(array(1, NULL, 3), x -> ) AS result
         """
       Then query result
-        | result       |
-        | [1, NULL, 3] |
+        | result   |
+        |  |
 
-    Scenario: Constant false predicate drops null elements
-      When query
-        """
-        SELECT filter(array(1, NULL, 3), x -> false) AS result
-        """
-      Then query result
-        | result |
-        | []     |
+      Examples:
+        | case                                         | pred  | result       |
+        | Constant true predicate keeps null elements  | true  | [1, NULL, 3] |
+        | Constant false predicate drops null elements | false | []           |
 
   Rule: Filter with additional predicate forms
 
-    Scenario: Filter with IN predicate
+    Scenario Outline: Additional predicate form: 
       When query
         """
-        SELECT filter(array(1, 2, 3, 4, 5), x -> x IN (1, 3, 5)) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result    |
-        | [1, 3, 5] |
-
-    Scenario: Filter with CASE WHEN in the predicate
-      When query
-        """
-        SELECT filter(array(1, 2, 3, 4), x -> CASE WHEN x % 2 = 0 THEN true ELSE false END) AS result
-        """
-      Then query result
-        | result |
-        | [2, 4] |
+        | result   |
+        |  |
 
-    Scenario: Filter with coalesce in the predicate over null elements
-      When query
-        """
-        SELECT filter(array(1, NULL, 3), x -> coalesce(x, 0) > 1) AS result
-        """
-      Then query result
-        | result |
-        | [3]    |
+      Examples:
+        | case                                                     | arr                  | pred                                         | result    |
+        | Filter with IN predicate                                 | array(1, 2, 3, 4, 5) | x IN (1, 3, 5)                               | [1, 3, 5] |
+        | Filter with CASE WHEN in the predicate                   | array(1, 2, 3, 4)    | CASE WHEN x % 2 = 0 THEN true ELSE false END | [2, 4]    |
+        | Filter with coalesce in the predicate over null elements | array(1, NULL, 3)    | coalesce(x, 0) > 1                           | [3]       |
 
   Rule: Filter with bigint arrays
 
@@ -710,23 +548,19 @@ Feature: array filter with lambda
 
   Rule: Filter with decimal and date arrays
 
-    Scenario: Filter decimal array
+    Scenario Outline: Decimal and date array: 
       When query
         """
-        SELECT filter(array(1.5BD, 2.7BD, 3.2BD), x -> x > 2.0BD) AS result
+        SELECT filter(, x -> ) AS result
         """
       Then query result
-        | result     |
-        | [2.7, 3.2] |
+        | result   |
+        |  |
 
-    Scenario: Filter date array
-      When query
-        """
-        SELECT filter(array(DATE '2020-01-01', DATE '2021-06-15', DATE '2019-03-03'), x -> x > DATE '2020-01-01') AS result
-        """
-      Then query result
-        | result         |
-        | [2021-06-15]   |
+      Examples:
+        | case                 | arr                                                            | pred                  | result       |
+        | Filter decimal array | array(1.5BD, 2.7BD, 3.2BD)                                     | x > 2.0BD             | [2.7, 3.2]   |
+        | Filter date array    | array(DATE '2020-01-01', DATE '2021-06-15', DATE '2019-03-03') | x > DATE '2020-01-01' | [2021-06-15] |
 
   Rule: Filter with index and a nested higher-order function
 
@@ -736,7 +570,7 @@ Feature: array filter with lambda
         SELECT filter(array(array(1, 2, 3), array(0), array(5, 6)), (x, i) -> size(filter(x, y -> y > i)) > 1) AS result
         """
       Then query result
-        | result             |
+        | result              |
         | [[1, 2, 3], [5, 6]] |
 
   Rule: Filter inside a join condition
@@ -762,44 +596,18 @@ Feature: array filter with lambda
 
   Rule: Invalid lambda functions
 
-    Scenario: Lambda with three parameters is rejected
-      When query
-        """
-        SELECT filter(array(1, 2, 3), (x, i, z) -> true) AS result
-        """
-      Then query error .*
-
-    Scenario: Lambda with non-boolean result is rejected
-      When query
-        """
-        SELECT filter(array(1, 2, 3), x -> x + 1) AS result
-        """
-      Then query error .*
-
-    Scenario: Lambda with duplicate parameter names is rejected
-      When query
-        """
-        SELECT filter(array(1, 2, 3), (x, x) -> x > 1) AS result
-        """
-      Then query error .*
-
-    Scenario: Lambda with case-insensitive duplicate parameter names is rejected
+    Scenario Outline: Invalid lambda: 
       When query
         """
-        SELECT filter(array(1, 2, 3), (x, X) -> x > 1) AS result
+        SELECT filter() AS result
         """
       Then query error .*
 
-    Scenario: Filter over a non-array first argument is rejected
-      When query
-        """
-        SELECT filter(42, x -> x > 0) AS result
-        """
-      Then query error .*
-
-    Scenario: Filter over a map first argument is rejected
-      When query
-        """
-        SELECT filter(map('a', 1), x -> x > 0) AS result
-        """
-      Then query error .*
+      Examples:
+        | case                                                               | args                              |
+        | Lambda with three parameters is rejected                           | array(1, 2, 3), (x, i, z) -> true |
+        | Lambda with non-boolean result is rejected                         | array(1, 2, 3), x -> x + 1        |
+        | Lambda with duplicate parameter names is rejected                  | array(1, 2, 3), (x, x) -> x > 1   |
+        | Lambda with case-insensitive duplicate parameter names is rejected | array(1, 2, 3), (x, X) -> x > 1   |
+        | Filter over a non-array first argument is rejected                 | 42, x -> x > 0                    |
+        | Filter over a map first argument is rejected                       | map('a', 1), x -> x > 0           |
diff --git a/python/pysail/tests/spark/function/features/array/array_insert.feature b/python/pysail/tests/spark/function/features/array/array_insert.feature
new file mode 100644
index 000000000..590d8780b
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_insert.feature
@@ -0,0 +1,93 @@
+Feature: array_insert with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: array_insert — the argument may come from a column
+
+    @function(columnargs)
+    Scenario: array_insert with the argument as a literal
+      When query
+        """
+        SELECT array_insert(array(1, 2, 3, 4), 5, 5) AS result
+        """
+      Then query result ordered
+        | result          |
+        | [1, 2, 3, 4, 5] |
+
+    # Sail rejects the column: Sail errors: array_insert: the index 0 is invalid. An index shall be either < 0 or > 0 (the first eleme...
+    @function(columnargs) @sail-bug
+    Scenario: array_insert takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT array_insert(array(1, 2, 3, 4), c, 5) AS result FROM VALUES (1, 5), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result          |
+        | [1, 2, 3, 4, 5] |
+        | NULL            |
+
+    @function(columnargs)
+    Scenario: array_insert takes argument 2 from a column holding two different values
+      When query
+        """
+        SELECT array_insert(array(1, 2, 3), c, 9) AS result FROM VALUES (1, 1), (2, 3) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result       |
+        | [9, 1, 2, 3] |
+        | [1, 2, 9, 3] |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null array literal yields a non-nullable array
+      When query
+        """
+        SELECT array_insert(array(1, 2), 1, 9) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
+
+    @sail-bug
+    Scenario: a non-null array column yields a non-nullable array
+      When query
+        """
+        SELECT array_insert(array(id), 1, 9) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: long (containsNull = true)
+        """
+
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT array_insert(c, 1, 9) AS result FROM VALUES (array(1, 2)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
+
+    @sail-bug
+    Scenario: nullable input elements propagate to the element nullability
+      When query
+        """
+        SELECT array_insert(c, 1, 9) AS result FROM VALUES (array(1, CAST(NULL AS INT))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_intersect.feature b/python/pysail/tests/spark/function/features/array/array_intersect.feature
new file mode 100644
index 000000000..b3c6fd188
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_intersect.feature
@@ -0,0 +1,221 @@
+Feature: array_intersect() returns common array elements without duplicates
+
+  Rule: Source-of-truth examples
+
+    Scenario Outline: array_intersect 
+      When query
+        """
+        SELECT sort_array(array_intersect(, )) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                     | left                  | right                     | result    |
+        | basic usage              | array('b', 'a', 'c')  | array('c', 'd', 'a', 'f') | [a, c]    |
+        | with all common elements | array('a', 'b', 'c')  | array('a', 'b', 'c')      | [a, b, c] |
+        | with null values         | array('a', 'b', NULL) | array('a', NULL, 'c')     | [NULL, a] |
+
+    # These two keep the raw (unsorted) result because it is already empty.
+    Scenario Outline: array_intersect  returns an empty array
+      When query
+        """
+        SELECT array_intersect(, ) AS result
+        """
+      Then query result
+        | result |
+        | []     |
+
+      Examples:
+        | case                    | left                           | right                |
+        | with no common elements | array('b', 'a', 'c')           | array('d', 'e', 'f') |
+        | with empty arrays       | CAST(array() AS ARRAY) | array('a', 'b', 'c') |
+
+    Scenario Outline: array_intersect on array columns 
+      When query
+        """
+        SELECT array_intersect(c1, c2) AS result
+        FROM VALUES
+          ()
+        AS t(c1, c2)
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                      | pair                                            | result |
+        | preserves first-array order               | array('b', 'a', 'c'), array('c', 'd', 'a', 'f') | [a, c] |
+        | preserves left order when left is shorter | array(1, 2), array(2, 1, 3)                     | [1, 2] |
+
+  Rule: Duplicates and multiple rows
+
+    Scenario: array_intersect removes duplicates from the result
+      When query
+        """
+        SELECT sort_array(array_intersect(array(1, 2, 2, 3, 3), array(2, 2, 3, 4))) AS result
+        """
+      Then query result
+        | result |
+        | [2, 3] |
+
+    Scenario: array_intersect on identical arrays deduplicates in left-array order
+      When query
+        """
+        SELECT array_intersect(c1, c2) AS result
+        FROM VALUES
+          (array('b', 'a', 'c', 'c'), array('b', 'a', 'c', 'c'))
+        AS t(c1, c2)
+        """
+      Then query result
+        | result    |
+        | [b, a, c] |
+
+    Scenario: array_intersect across multiple rows
+      When query
+        """
+        SELECT id, sort_array(array_intersect(left_arr, right_arr)) AS result
+        FROM VALUES
+          (1, array(1, 2, 3), array(2, 3, 4)),
+          (2, array(1, 1, 1), array(1, 2)),
+          (3, array(1, 2), array(3, 4)),
+          (4, CAST(NULL AS ARRAY), array(1, 2))
+        AS t(id, left_arr, right_arr)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | result |
+        | 1  | [2, 3] |
+        | 2  | [1]    |
+        | 3  | []     |
+        | 4  | NULL   |
+
+  Rule: Type-specific behavior
+
+    Scenario Outline: array_intersect with  arrays
+      When query
+        """
+        SELECT sort_array(array_intersect(, )) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case    | left                     | right                   | result     |
+        | boolean | array(true, false, true) | array(false, false)     | [false]    |
+        | double  | array(1.5D, 2.5D, 3.5D)  | array(3.5D, 4.5D, 1.5D) | [1.5, 3.5] |
+
+    Scenario: array_intersect with arrays of structs
+      When query
+        """
+        SELECT array_intersect(
+          array(named_struct('id', 1, 'name', 'a'), named_struct('id', 2, 'name', 'b')),
+          array(named_struct('id', 2, 'name', 'b'), named_struct('id', 3, 'name', 'c'))
+        ) AS result
+        """
+      Then query result
+        | result   |
+        | [{2, b}] |
+
+    Scenario: array_intersect with array element type
+      When query
+        """
+        SELECT id, array_intersect(left_arr, right_arr) AS result
+        FROM VALUES
+          (1, array(NULL), array(NULL, NULL)),
+          (2, array(NULL), array()),
+          (3, array(NULL, NULL), array(NULL))
+        AS t(id, left_arr, right_arr)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | result |
+        | 1  | [NULL] |
+        | 2  | []     |
+        | 3  | [NULL] |
+
+    Scenario: array elements do not match typed non-null elements
+      When query
+        """
+        SELECT id, array_intersect(left_arr, right_arr) AS result
+        FROM VALUES
+          (1, array(NULL, NULL), array('d', 'e', 'f')),
+          (2, array(NULL),       CAST(array() AS ARRAY)),
+          (3, array(),           CAST(array() AS ARRAY))
+        AS t(id, left_arr, right_arr)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | result |
+        | 1  | []     |
+        | 2  | []     |
+        | 3  | []     |
+
+  Rule: Invalid inputs
+
+    Scenario Outline: array_intersect rejects 
+      When query
+        """
+        SELECT array_intersect() AS result
+        """
+      Then query error .*
+
+      Examples:
+        | case                       | args                 |
+        | non-array inputs           | 1, array(1)          |
+        | incompatible element types | array(1), array('1') |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: non-null array literals yield a non-nullable array
+      When query
+        """
+        SELECT array_intersect(array(1, 2), array(2, 3)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a non-null array column yields a non-nullable array
+      When query
+        """
+        SELECT array_intersect(array(id), array(id)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: long (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT array_intersect(c, array(2)) AS result FROM VALUES (array(1, 2)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: nullable input elements propagate to the element nullability
+      When query
+        """
+        SELECT array_intersect(c, array(1)) AS result FROM VALUES (array(1, CAST(NULL AS INT))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_join.feature b/python/pysail/tests/spark/function/features/array/array_join.feature
new file mode 100644
index 000000000..8718811f6
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_join.feature
@@ -0,0 +1,27 @@
+Feature: array_join output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_join yields the schema Spark declares
+      When query
+        """
+        SELECT array_join(array('hello', 'world'), ' ') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
+
+    Scenario: a nullable column input to array_join stays nullable
+      When query
+        """
+        SELECT array_join(c, ' ') AS result FROM VALUES (array('hello', 'world')), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_max.feature b/python/pysail/tests/spark/function/features/array/array_max.feature
new file mode 100644
index 000000000..cc1d1a9fe
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_max.feature
@@ -0,0 +1,26 @@
+Feature: array_max output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to array_max yields the schema Spark declares
+      When query
+        """
+        SELECT array_max(array(1, 20, null, 3)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a nullable column input to array_max stays nullable
+      When query
+        """
+        SELECT array_max(c) AS result FROM VALUES (array(1, 20, null, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_min.feature b/python/pysail/tests/spark/function/features/array/array_min.feature
new file mode 100644
index 000000000..480cd778d
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_min.feature
@@ -0,0 +1,26 @@
+Feature: array_min output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to array_min yields the schema Spark declares
+      When query
+        """
+        SELECT array_min(array(1, 20, null, 3)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a nullable column input to array_min stays nullable
+      When query
+        """
+        SELECT array_min(c) AS result FROM VALUES (array(1, 20, null, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_min_max.feature b/python/pysail/tests/spark/function/features/array/array_min_max.feature
new file mode 100644
index 000000000..ef1aaf0e2
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_min_max.feature
@@ -0,0 +1,291 @@
+Feature: array_min and array_max functions
+
+  Rule: Basic usage
+
+    Scenario Outline: Basic: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                           | arr                                | min   | max    |
+        | array_min and array_max with integers          | array(3, 1, 2)                     | 1     | 3      |
+        | array_min and array_max with strings           | array('banana', 'apple', 'cherry') | apple | cherry |
+        | array_min and array_max with doubles           | array(3.14, 1.5, 2.7)              | 1.50  | 3.14   |
+        | array_min and array_max with single element    | array(42)                          | 42    | 42     |
+        | array_min and array_max with repeated elements | array(5, 5, 5)                     | 5     | 5      |
+
+  Rule: Empty and NULL inputs
+
+    Scenario Outline: Empty and NULL: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                                | arr                                         | min  | max  |
+        | array_min and array_max with empty array            | array()                                     | NULL | NULL |
+        | array_min and array_max with NULL input             | NULL                                        | NULL | NULL |
+        | array_min and array_max with all NULLs              | array(CAST(NULL AS INT), CAST(NULL AS INT)) | NULL | NULL |
+        | array_min and array_max with some NULLs mixed       | array(3, NULL, 1, NULL, 2)                  | 1    | 3    |
+        | array_min and array_max with NULL at first position | array(NULL, 2, 3)                           | 2    | 3    |
+        | array_min and array_max with NULL at last position  | array(1, 2, NULL)                           | 1    | 2    |
+
+  Rule: Negative numbers
+
+    Scenario Outline: Negative: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                                          | arr                | min | max |
+        | array_min and array_max with negative numbers                 | array(-5, -1, -10) | -10 | -1  |
+        | array_min and array_max with mixed positive negative and zero | array(-3, 0, 3)    | -3  | 3   |
+
+  Rule: Float special values
+
+    Scenario Outline: Float special: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                                    | arr                                                            | min       | max      |
+        | array_min and array_max with NaN                        | array(1.0, CAST('NaN' AS DOUBLE), 2.0)                         | 1.0       | NaN      |
+        | array_min and array_max with NaN only                   | array(CAST('NaN' AS DOUBLE))                                   | NaN       | NaN      |
+        | array_min and array_max with NaN and NULL               | array(CAST('NaN' AS DOUBLE), NULL, 1.0)                        | 1.0       | NaN      |
+        | array_min and array_max with NaN vs Infinity            | array(CAST('NaN' AS DOUBLE), CAST('Infinity' AS DOUBLE))       | Infinity  | NaN      |
+        | array_min and array_max with NaN vs negative Infinity   | array(CAST('-Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE))      | -Infinity | NaN      |
+        | array_min and array_max with positive Infinity          | array(1.0, CAST('Infinity' AS DOUBLE))                         | 1.0       | Infinity |
+        | array_min and array_max with negative Infinity          | array(1.0, CAST('-Infinity' AS DOUBLE))                        | -Infinity | 1.0      |
+        | array_min and array_max with both Infinities            | array(CAST('-Infinity' AS DOUBLE), CAST('Infinity' AS DOUBLE)) | -Infinity | Infinity |
+        | array_min and array_max with float NaN                  | array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT))                | 1.0       | NaN      |
+        | array_min and array_max with positive and negative zero | array(CAST(0.0 AS DOUBLE), CAST(-0.0 AS DOUBLE))               | 0.0       | 0.0      |
+
+    @sail-only
+    Scenario: array_min and array_max with extreme double values (display format differs from Spark)
+      When query
+        """
+        SELECT array_min(array(1.7976931348623157E308, -1.7976931348623157E308, 0.0)) AS min_val, array_max(array(1.7976931348623157E308, -1.7976931348623157E308, 0.0)) AS max_val
+        """
+      Then query result
+        | min_val                 | max_val                |
+        | -1.7976931348623157e308 | 1.7976931348623157e308 |
+
+  Rule: Boolean arrays
+
+    Scenario: array_min and array_max with booleans
+      When query
+        """
+        SELECT array_min(array(true, false)) AS min_val, array_max(array(true, false)) AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        | false   | true    |
+
+  Rule: String edge cases
+
+    Scenario: array_min and array_max with empty string in array
+      When query
+        """
+        SELECT array_min(array('', 'a', 'b')) AS min_val, array_max(array('', 'a', 'b')) AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |         | b       |
+
+    Scenario Outline: String: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                                        | arr                        | min | max |
+        | array_min and array_max with case sensitive strings         | array('Z', 'a', 'A')       | A   | a   |
+        | array_min and array_max with mixed case and numeric strings | array('abc', 'ABc', '123') | 123 | abc |
+
+  Rule: Date and timestamp arrays
+
+    Scenario Outline: Date/timestamp: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                        | arr                                                                                                      | min                 | max                 |
+        | array_min and array_max with dates          | array(DATE '2023-01-01', DATE '2023-12-31', DATE '2023-06-15')                                           | 2023-01-01          | 2023-12-31          |
+        | array_min and array_max with timestamps     | array(TIMESTAMP '2023-01-01 00:00:00', TIMESTAMP '2023-12-31 23:59:59', TIMESTAMP '2023-06-15 12:00:00') | 2023-01-01 00:00:00 | 2023-12-31 23:59:59 |
+        | array_min and array_max with dates and NULL | array(DATE '2023-01-01', NULL, DATE '2023-12-31')                                                        | 2023-01-01          | 2023-12-31          |
+
+  Rule: Decimal arrays
+
+    Scenario: array_min and array_max with decimals
+      When query
+        """
+        SELECT array_min(array(CAST(1.11 AS DECIMAL(10,2)), CAST(2.22 AS DECIMAL(10,2)), CAST(0.99 AS DECIMAL(10,2)))) AS min_val, array_max(array(CAST(1.11 AS DECIMAL(10,2)), CAST(2.22 AS DECIMAL(10,2)), CAST(0.99 AS DECIMAL(10,2)))) AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        | 0.99    | 2.22    |
+
+    Scenario: array_min with equal decimal values of different precision
+      When query
+        """
+        SELECT array_min(array(CAST(0.1 AS DECIMAL(38,18)), CAST(0.10 AS DECIMAL(38,18)), CAST(0.100 AS DECIMAL(38,18)))) AS min_val
+        """
+      Then query result
+        | min_val              |
+        | 0.100000000000000000 |
+
+  Rule: BIGINT boundary values
+
+    Scenario Outline: Integer extremes: 
+      When query
+        """
+        SELECT array_min() AS min_val, array_max() AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        |    |    |
+
+      Examples:
+        | case                                           | arr                                                                                                 | min                  | max                 |
+        | array_min and array_max with BIGINT extremes   | array(CAST(9223372036854775807 AS BIGINT), CAST(-9223372036854775808 AS BIGINT), CAST(0 AS BIGINT)) | -9223372036854775808 | 9223372036854775807 |
+        | array_min and array_max with SMALLINT extremes | array(CAST(1 AS SMALLINT), CAST(-32768 AS SMALLINT), CAST(32767 AS SMALLINT))                       | -32768               | 32767               |
+        | array_min and array_max with TINYINT extremes  | array(CAST(1 AS TINYINT), CAST(-128 AS TINYINT), CAST(127 AS TINYINT))                              | -128                 | 127                 |
+
+  Rule: Large arrays
+
+    Scenario: array_min and array_max with large sequence
+      When query
+        """
+        SELECT array_min(sequence(1, 1000)) AS min_val, array_max(sequence(1, 1000)) AS max_val
+        """
+      Then query result
+        | min_val | max_val |
+        | 1       | 1000    |
+
+  Rule: Nested arrays
+
+    Scenario Outline:  with nested arrays
+      When query
+        """
+        SELECT (array(array(1,2), array(3,4))) AS 
+        """
+      Then query result
+        |   |
+        |  |
+
+      Examples:
+        | fn        | alias   | result |
+        | array_min | min_val | [1, 2] |
+        | array_max | max_val | [3, 4] |
+
+  Rule: Multi-row results
+
+    Scenario: array_min and array_max across multiple rows
+      When query
+        """
+        SELECT id, array_min(arr) AS min_val, array_max(arr) AS max_val
+        FROM VALUES (1, array(10, 20, 30)), (2, array(5, 15, 25)), (3, array(100)) AS t(id, arr)
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | min_val | max_val |
+        | 1  | 10      | 30      |
+        | 2  | 5       | 25      |
+        | 3  | 100     | 100     |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: array_min of a non-null array literal
+      When query
+        """
+        SELECT array_min(array(3, 1, 2)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: array_min of a non-null array column
+      When query
+        """
+        SELECT array_min(array(id, id)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: array_min of a nullable array column stays nullable
+      When query
+        """
+        SELECT array_min(c) AS result FROM VALUES (array(1, 2)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: array_max of a non-null array literal
+      When query
+        """
+        SELECT array_max(array(3, 1, 2)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: array_max of a non-null array column
+      When query
+        """
+        SELECT array_max(array(id, id)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: array_max of a nullable array column stays nullable
+      When query
+        """
+        SELECT array_max(c) AS result FROM VALUES (array(1, 2)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_position.feature b/python/pysail/tests/spark/function/features/array/array_position.feature
new file mode 100644
index 000000000..d6edb372c
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_position.feature
@@ -0,0 +1,27 @@
+Feature: array_position output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_position yields the schema Spark declares
+      When query
+        """
+        SELECT array_position(array(312, 773, 708, 708), 708) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = false)
+        """
+
+    Scenario: a nullable column input to array_position stays nullable
+      When query
+        """
+        SELECT array_position(c, 708) AS result FROM VALUES (array(312, 773, 708, 708)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_prepend.feature b/python/pysail/tests/spark/function/features/array/array_prepend.feature
new file mode 100644
index 000000000..ed654790b
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_prepend.feature
@@ -0,0 +1,29 @@
+Feature: array_prepend output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_prepend yields the schema Spark declares
+      When query
+        """
+        SELECT array_prepend(array('b', 'd', 'c', 'a'), 'd') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = true)
+        """
+
+    Scenario: a nullable column input to array_prepend stays nullable
+      When query
+        """
+        SELECT array_prepend(c, 'd') AS result FROM VALUES (array('b', 'd', 'c', 'a')), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: string (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_remove.feature b/python/pysail/tests/spark/function/features/array/array_remove.feature
new file mode 100644
index 000000000..89e1aef2e
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_remove.feature
@@ -0,0 +1,91 @@
+Feature: array_remove with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: array_remove — the argument may come from a column
+
+    @function(columnargs)
+    Scenario: array_remove with the argument as a literal
+      When query
+        """
+        SELECT array_remove(array(1, 2, 3, null, 3), 3) AS result
+        """
+      Then query result ordered
+        | result       |
+        | [1, 2, NULL] |
+
+    # Sail rejects the column: Sail errors: Invalid argument error: Column '#4' is declared as non-nullable but contains null values
+    @function(columnargs) @sail-bug
+    Scenario: array_remove takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT array_remove(array(1, 2, 3, null, 3), c) AS result FROM VALUES (1, 3), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result       |
+        | [1, 2, NULL] |
+        | NULL         |
+
+    @function(columnargs)
+    Scenario: array_remove takes argument 2 from a column holding two different values
+      When query
+        """
+        SELECT array_remove(array(1, 2, 3), c) AS result FROM VALUES (1, 1), (2, 3) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result |
+        | [2, 3] |
+        | [1, 2] |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null array literal yields a non-nullable array
+      When query
+        """
+        SELECT array_remove(array(1, 2, 3), 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a non-null array column yields a non-nullable array
+      When query
+        """
+        SELECT array_remove(array(id), 2) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: long (containsNull = false)
+        """
+
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT array_remove(c, 2) AS result FROM VALUES (array(1, 2)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
+
+    @sail-bug
+    Scenario: nullable input elements propagate to the element nullability
+      When query
+        """
+        SELECT array_remove(c, 2) AS result FROM VALUES (array(1, CAST(NULL AS INT))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_repeat.feature b/python/pysail/tests/spark/function/features/array/array_repeat.feature
new file mode 100644
index 000000000..779e34234
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_repeat.feature
@@ -0,0 +1,43 @@
+Feature: array_repeat output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_repeat yields the schema Spark declares
+      When query
+        """
+        SELECT array_repeat('123', 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null column input to array_repeat yields the schema Spark declares
+      When query
+        """
+        SELECT array_repeat(CAST(id AS STRING), 2) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a nullable column input to array_repeat stays nullable
+      When query
+        """
+        SELECT array_repeat(c, 2) AS result FROM VALUES ('123'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/array_sort.feature b/python/pysail/tests/spark/function/features/array/array_sort.feature
new file mode 100644
index 000000000..40ae927b8
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_sort.feature
@@ -0,0 +1,106 @@
+Feature: array_sort higher-order function
+
+  Rule: No-comparator form — natural ascending order, NULLs last
+
+    Scenario Outline: No-comparator form: 
+      When query
+        """
+        SELECT array_sort() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                 | arr                                                 | result                     |
+        | Sort integers ascending                              | array(3, 1, 2)                                      | [1, 2, 3]                  |
+        | Sort integers with a null — null placed last         | array(3, NULL, 1)                                   | [1, 3, NULL]               |
+        | Sort strings ascending                               | array('b', 'a', 'c')                                | [a, b, c]                  |
+        | Sort doubles with NaN and Infinity — NaN is greatest | array(double('NaN'), 1.0, double('Infinity'), -1.0) | [-1.0, 1.0, Infinity, NaN] |
+        | Sort null array returns null                         | CAST(NULL AS ARRAY)                            | NULL                       |
+
+  Rule: Comparator form — ascending and descending
+
+    Scenario Outline: Comparator direction: 
+      When query
+        """
+        SELECT array_sort(array(5, 6, 1), ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                      | cmp                                                            | result    |
+        | Ascending comparator with case expression | (l, r) -> CASE WHEN l < r THEN -1 WHEN l > r THEN 1 ELSE 0 END | [1, 5, 6] |
+        | Ascending comparator with subtraction     | (l, r) -> l - r                                                | [1, 5, 6] |
+        | Descending comparator with subtraction    | (l, r) -> r - l                                                | [6, 5, 1] |
+
+  Rule: Comparator form — element types
+
+    Scenario: Comparator over array of structs by field
+      When query
+        """
+        SELECT array_sort(array(named_struct('x', 3), named_struct('x', 1), named_struct('x', 2)), (l, r) -> l.x - r.x) AS result
+        """
+      Then query result
+        | result          |
+        | [{1}, {2}, {3}] |
+
+    Scenario: Comparator over strings with explicit null handling — nulls first
+      When query
+        """
+        SELECT array_sort(array('bc', 'ab', NULL, 'dc'), (l, r) -> CASE WHEN l IS NULL AND r IS NULL THEN 0 WHEN l IS NULL THEN -1 WHEN r IS NULL THEN 1 WHEN l < r THEN 1 WHEN l > r THEN -1 ELSE 0 END) AS result
+        """
+      Then query result
+        | result             |
+        | [NULL, dc, bc, ab] |
+
+  Rule: Comparator form — degenerate inputs
+
+    Scenario Outline: Degenerate input: 
+      When query
+        """
+        SELECT array_sort(, (l, r) -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                       | arr                         | cmp   | result    |
+        | Comparator on single-element array         | array(5)                    | l - r | [5]       |
+        | Comparator on empty array                  | CAST(array() AS ARRAY) | l - r | []        |
+        | Comparator on null array returns null      | CAST(NULL AS ARRAY)    | l - r | NULL      |
+        | Constant-zero comparator keeps input order | array(3, 1, 2)              | 0     | [3, 1, 2] |
+
+  Rule: Comparator form — multi-row
+
+    Scenario: Comparator applied per row
+      When query
+        """
+        SELECT array_sort(a, (l, r) -> l - r) AS result
+        FROM VALUES (array(3, 1, 2)), (array(9, 7, 8)), (CAST(NULL AS ARRAY)) AS t(a)
+        """
+      Then query result
+        | result    |
+        | [1, 2, 3] |
+        | [7, 8, 9] |
+        | NULL      |
+
+  Rule: Comparator form — errors
+
+    Scenario Outline: Error case: 
+      When query
+        """
+        SELECT array_sort() AS result
+        """
+      Then query error .*
+
+      Examples:
+        | case                                                | args                                         |
+        | Comparator returning null raises an error           | array(2, 1), (l, r) -> CAST(NULL AS INT)     |
+        | Comparator returning a non-integer type is rejected | array(2, 1), (l, r) -> CAST(l - r AS BIGINT) |
+        | Comparator with one parameter is rejected           | array(2, 1), x -> x                          |
+        | Comparator with three parameters is rejected        | array(2, 1), (a, b, c) -> 1                  |
+        | Non-array first argument is rejected                | 5, (l, r) -> 1                               |
diff --git a/python/pysail/tests/spark/function/features/array/array_union.feature b/python/pysail/tests/spark/function/features/array/array_union.feature
new file mode 100644
index 000000000..f738a7eb6
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/array_union.feature
@@ -0,0 +1,29 @@
+Feature: array_union output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to array_union yields the schema Spark declares
+      When query
+        """
+        SELECT array_union(array(1, 2, 3), array(1, 3, 5)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a nullable column input to array_union stays nullable
+      When query
+        """
+        SELECT array_union(c, array(1, 3, 5)) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array_update.feature b/python/pysail/tests/spark/function/features/array/array_update.feature
similarity index 100%
rename from python/pysail/tests/spark/function/features/array_update.feature
rename to python/pysail/tests/spark/function/features/array/array_update.feature
diff --git a/python/pysail/tests/spark/function/features/array/arrays_overlap.feature b/python/pysail/tests/spark/function/features/array/arrays_overlap.feature
new file mode 100644
index 000000000..0d2aa55da
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/arrays_overlap.feature
@@ -0,0 +1,27 @@
+Feature: arrays_overlap output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to arrays_overlap yields the schema Spark declares
+      When query
+        """
+        SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = false)
+        """
+
+    Scenario: a nullable column input to arrays_overlap stays nullable
+      When query
+        """
+        SELECT arrays_overlap(c, array(3, 4, 5)) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/arrays_zip.feature b/python/pysail/tests/spark/function/features/array/arrays_zip.feature
new file mode 100644
index 000000000..39450b8c4
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/arrays_zip.feature
@@ -0,0 +1,418 @@
+Feature: arrays_zip comprehensive tests
+
+  Rule: Basic usage
+
+    Scenario Outline: Basic: 
+      When query
+        """
+        SELECT arrays_zip() AS result 
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                     | args                                                           | from                               | result                                                                |
+        | arrays_zip two arrays same length        | array(1,2,3), array('a','b','c')                               |                                    | [{1, a}, {2, b}, {3, c}]                                              |
+        | arrays_zip three arrays                  | array(1,2), array('a','b'), array(true,false)                  |                                    | [{1, a, true}, {2, b, false}]                                         |
+        | arrays_zip single array                  | array(1,2,3)                                                   |                                    | [{1}, {2}, {3}]                                                       |
+        | arrays_zip four args asymmetric          | array(1), array('a','b'), array(true, false, NULL), array(1.0) |                                    | [{1, a, true, 1.0}, {NULL, b, false, NULL}, {NULL, NULL, NULL, NULL}] |
+        | arrays_zip self-zip same column          | a, a                                                           | FROM VALUES (array(1,2,3)) AS t(a) | [{1, 1}, {2, 2}, {3, 3}]                                              |
+        | arrays_zip zero args returns empty array |                                                                |                                    | []                                                                    |
+
+  Rule: Different array lengths
+
+    Scenario Outline: Different lengths: 
+      When query
+        """
+        SELECT arrays_zip() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                               | args                                 | result                                               |
+        | arrays_zip first longer pads NULL  | array(1,2,3), array('a','b')         | [{1, a}, {2, b}, {3, NULL}]                          |
+        | arrays_zip second longer pads NULL | array(1,2), array('a','b','c')       | [{1, a}, {2, b}, {NULL, c}]                          |
+        | arrays_zip one empty array         | array(1,2), array()                  | [{1, NULL}, {2, NULL}]                               |
+        | arrays_zip both empty arrays       | array(), array()                     | []                                                   |
+        | arrays_zip very asymmetric 1 vs 5  | array(1), array('a','b','c','d','e') | [{1, a}, {NULL, b}, {NULL, c}, {NULL, d}, {NULL, e}] |
+
+    Scenario: arrays_zip four args all empty returns empty
+      When query
+        """
+        SELECT arrays_zip(
+          CAST(array() AS ARRAY),
+          CAST(array() AS ARRAY),
+          CAST(array() AS ARRAY),
+          CAST(array() AS ARRAY)
+        ) AS result
+        """
+      Then query result
+        | result |
+        | []     |
+
+  Rule: NULL handling
+
+    Scenario Outline: NULL input: 
+      When query
+        """
+        SELECT arrays_zip() AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case                                       | args                                                     |
+        | arrays_zip untyped NULL array returns NULL | NULL, array(1,2)                                         |
+        | arrays_zip typed NULL array returns NULL   | CAST(NULL AS ARRAY), array(1,2)                     |
+        | arrays_zip both args NULL returns NULL     | CAST(NULL AS ARRAY), CAST(NULL AS ARRAY)    |
+        | arrays_zip NULL and empty returns NULL     | CAST(NULL AS ARRAY), CAST(array() AS ARRAY) |
+
+    Scenario: arrays_zip four NULL args returns NULL
+      When query
+        """
+        SELECT arrays_zip(
+          CAST(NULL AS ARRAY),
+          CAST(NULL AS ARRAY),
+          CAST(NULL AS ARRAY),
+          CAST(NULL AS ARRAY)
+        ) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario Outline: NULL elements: 
+      When query
+        """
+        SELECT arrays_zip() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                | args                                | result                       |
+        | arrays_zip with NULL elements                       | array(1,NULL,3), array('a','b','c') | [{1, a}, {NULL, b}, {3, c}]  |
+        | arrays_zip all NULL elements                        | array(NULL,NULL), array(NULL,NULL)  | [{NULL, NULL}, {NULL, NULL}] |
+        | arrays_zip untyped empty array() pads first as NULL | array(), array(1,2)                 | [{NULL, 1}, {NULL, 2}]       |
+
+    # Columnar path: neither column is fully NULL, so the invoke short-circuits do
+    # not fire — the combined validity mask makes every row NULL and the result
+    # values struct ends up empty. This exercises the flatten build with empty
+    # `take` outputs (the path that replaced the removed all-null special case).
+    Scenario: arrays_zip all rows NULL via combined validity across columns
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result
+        FROM VALUES
+          (array(1), CAST(NULL AS ARRAY)),
+          (CAST(NULL AS ARRAY), array(2))
+        AS t(a, b)
+        """
+      Then query result ordered
+        | result |
+        | NULL   |
+        | NULL   |
+
+    Scenario: arrays_zip mixed valid and NULL rows via a per-row NULL array
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result
+        FROM VALUES
+          (array(1,2), array('x','y')),
+          (array(3), CAST(NULL AS ARRAY))
+        AS t(a, b)
+        """
+      Then query result ordered
+        | result           |
+        | [{1, x}, {2, y}] |
+        | NULL             |
+
+  Rule: Columnar multi-row paths (flatten build)
+    # Multi-row FROM VALUES columns exercise the flatten kernel's per-row offset
+    # and null-pad logic that single-row literals never reach.
+
+    Scenario Outline: Columnar: 
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result
+        FROM VALUES  AS t(a, b)
+        """
+      Then query result ordered
+        | result |
+        |  |
+        |  |
+
+      Examples:
+        | case                                            | values                                                     | row1                           | row2                           |
+        | arrays_zip empty rows in a column               | (array(), array()), (array(), array())                     | []                             | []                             |
+        | arrays_zip ragged lengths per row in a column   | (array(1,2,3), array('a')), (array(4), array('b','c','d')) | [{1, a}, {2, NULL}, {3, NULL}] | [{4, b}, {NULL, c}, {NULL, d}] |
+        | arrays_zip empty and non-empty rows in a column | (array(), array(1)), (array(2), array())                   | [{NULL, 1}]                    | [{2, NULL}]                    |
+
+  Rule: Type variety
+
+    Scenario Outline: Type variety: 
+      When query
+        """
+        SELECT arrays_zip() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                            | args                                                                         | result                                       |
+        | arrays_zip int and double       | array(1,2), array(1.5,2.5)                                                   | [{1, 1.5}, {2, 2.5}]                         |
+        | arrays_zip nested arrays        | array(array(1,2),array(3,4)), array('a','b')                                 | [{[1, 2], a}, {[3, 4], b}]                   |
+        | arrays_zip boolean and int      | array(true,false), array(1,2)                                                | [{true, 1}, {false, 2}]                      |
+        | arrays_zip struct elements      | array(struct(1 AS a)), array(struct('x' AS b))                               | [{{1}, {x}}]                                 |
+        | arrays_zip map elements         | array(map(1,'a')), array(map(2,'b'))                                         | [{{1 -> a}, {2 -> b}}]                       |
+        | arrays_zip binary elements      | array(X'01', X'02'), array(X'0A', X'0B')                                     | [{[01], [0A]}, {[02], [0B]}]                 |
+        | arrays_zip date elements        | array(DATE'2024-01-01'), array(DATE'2024-12-31')                             | [{2024-01-01, 2024-12-31}]                   |
+        | arrays_zip timestamp elements   | array(TIMESTAMP'2024-01-01 00:00:00'), array(TIMESTAMP'2024-12-31 23:59:59') | [{2024-01-01 00:00:00, 2024-12-31 23:59:59}] |
+        | arrays_zip deeply nested arrays | array(array(1,2)), array(array(3,4))                                         | [{[1, 2], [3, 4]}]                           |
+        | arrays_zip sequence results     | sequence(1, 3), sequence(10, 12)                                             | [{1, 10}, {2, 11}, {3, 12}]                  |
+
+    Scenario: arrays_zip decimal elements
+      When query
+        """
+        SELECT arrays_zip(
+          array(CAST(1.5 AS DECIMAL(10,2)), CAST(2.5 AS DECIMAL(10,2))),
+          array(1, 2)
+        ) AS result
+        """
+      Then query result
+        | result                 |
+        | [{1.50, 1}, {2.50, 2}] |
+
+    Scenario: arrays_zip float32 elements
+      When query
+        """
+        SELECT arrays_zip(
+          array(CAST(1.5 AS FLOAT)),
+          array(CAST(2.5 AS FLOAT))
+        ) AS result
+        """
+      Then query result
+        | result       |
+        | [{1.5, 2.5}] |
+
+  Rule: Composition
+
+    Scenario: arrays_zip nested in arrays_zip
+      When query
+        """
+        SELECT arrays_zip(arrays_zip(array(1,2), array('a','b')), array(true, false)) AS result
+        """
+      Then query result
+        | result                            |
+        | [{{1, a}, true}, {{2, b}, false}] |
+
+    Scenario Outline: arrays_zip element field access position 
+      When query
+        """
+        SELECT arrays_zip(array(1,2,3), array('x','y','z'))[1].`` AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | field | result |
+        | 0     | 2      |
+        | 1     | y      |
+
+    Scenario: arrays_zip to_json roundtrip
+      When query
+        """
+        SELECT to_json(arrays_zip(array(1,2), array('a','b'))) AS result
+        """
+      Then query result
+        | result                            |
+        | [{"0":1,"1":"a"},{"0":2,"1":"b"}] |
+
+    Scenario: arrays_zip LATERAL VIEW explode flatten
+      When query
+        """
+        SELECT e.`0` AS x, e.`1` AS y FROM
+          (SELECT arrays_zip(array(1,2,3), array('a','b','c')) AS z) LATERAL VIEW explode(z) AS e
+        """
+      Then query result ordered
+        | x | y |
+        | 1 | a |
+        | 2 | b |
+        | 3 | c |
+
+  Rule: Multi-row
+
+    Scenario: arrays_zip multi-row
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result FROM VALUES (array(1,2), array('x','y')), (array(3), array('z','w')), (CAST(NULL AS ARRAY), array('a')) AS t(a, b)
+        """
+      Then query result
+        | result              |
+        | [{1, x}, {2, y}]    |
+        | [{3, z}, {NULL, w}] |
+        | NULL                |
+
+    Scenario: arrays_zip multi-row uneven lengths per row
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result FROM VALUES
+          (array(1,2,3), array('x','y')),
+          (array(1), array('a','b','c','d','e')),
+          (array(), array('z'))
+        AS t(a, b)
+        """
+      Then query result
+        | result                                               |
+        | [{1, x}, {2, y}, {3, NULL}]                          |
+        | [{1, a}, {NULL, b}, {NULL, c}, {NULL, d}, {NULL, e}] |
+        | [{NULL, z}]                                          |
+
+    Scenario: arrays_zip multi-row empty row mixed with NULL
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result FROM VALUES
+          (array(), CAST(array() AS ARRAY)),
+          (array(1), array('x')),
+          (CAST(NULL AS ARRAY), CAST(NULL AS ARRAY))
+        AS t(a, b)
+        """
+      Then query result
+        | result   |
+        | []       |
+        | [{1, x}] |
+        | NULL     |
+
+    Scenario: arrays_zip multi-row all-null column returns all NULL
+      When query
+        """
+        SELECT arrays_zip(a, b) AS result FROM VALUES
+          (CAST(NULL AS ARRAY), array(1, 2)),
+          (CAST(NULL AS ARRAY), array(3, 4)),
+          (CAST(NULL AS ARRAY), array(5, 6))
+        AS t(a, b)
+        """
+      Then query result
+        | result |
+        | NULL   |
+        | NULL   |
+        | NULL   |
+
+  Rule: Error conditions
+
+    Scenario Outline: Error: 
+      When query
+        """
+        SELECT arrays_zip() AS result
+        """
+      Then query error .*
+
+      Examples:
+        | case                                        | args                |
+        | arrays_zip non-array input errors           | 1, 2                |
+        | arrays_zip mixed array and non-array errors | array(1,2), 'hello' |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: non-null array literals yield a non-nullable array
+      When query
+        """
+        SELECT arrays_zip(array(1, 2), array(3, 4)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: struct (containsNull = false)
+         |    |    |-- 0: integer (nullable = true)
+         |    |    |-- 1: integer (nullable = true)
+        """
+
+    @sail-bug
+    Scenario: a non-null array column yields a non-nullable array
+      When query
+        """
+        SELECT arrays_zip(array(id), array(id)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: struct (containsNull = false)
+         |    |    |-- 0: long (nullable = true)
+         |    |    |-- 1: long (nullable = true)
+        """
+
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT arrays_zip(c, c) AS result FROM VALUES (array(1)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: struct (containsNull = false)
+         |    |    |-- c: integer (nullable = true)
+         |    |    |-- c: integer (nullable = true)
+        """
+
+    @sail-bug
+    Scenario: nullable input elements propagate into the struct fields
+      When query
+        """
+        SELECT arrays_zip(c, c) AS result FROM VALUES (array(1, CAST(NULL AS INT))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: struct (containsNull = false)
+         |    |    |-- c: integer (nullable = true)
+         |    |    |-- c: integer (nullable = true)
+        """
+
+  Rule: Result values (migrated from test_arrays_zip.txt doctests)
+
+    Scenario Outline: Result values: 
+      When query
+        """
+        SELECT arrays_zip() AS  FROM VALUES () AS t()
+        """
+      Then query result
+        |   |
+        |  |
+
+      Examples:
+        | case                           | cols                 | alias  | values                                              | result                               |
+        | arrays_zip doctest #1 (result) | vals1, vals2, vals3  | zipped | array(1L, 2L, 3L), array(2L, 4L, 6L), array(3L, 6L) | [{1, 2, 3}, {2, 4, 6}, {3, 6, NULL}] |
+        | arrays_zip doctest #3 (result) | nums, letters        | r      | array(1, 2, 3), array('a', 'b', 'c')                | [{1, a}, {2, b}, {3, c}]             |
+        | arrays_zip doctest #4 (result) | nums, letters        | r      | array(1, 2), array('a', 'b', 'c')                   | [{1, a}, {2, b}, {NULL, c}]          |
+        | arrays_zip doctest #5 (result) | nums, letters, bools | r      | array(1, 2), array('a', 'b'), array(true, false)    | [{1, a, true}, {2, b, false}]        |
+        | arrays_zip doctest #6 (result) | nums, letters        | r      | array(1, 2, NULL), array('a', NULL, 'c')            | [{1, a}, {2, NULL}, {NULL, c}]       |
+
+  Rule: Output schema (migrated from test_arrays_zip.txt printSchema doctests)
+
+    Scenario: arrays_zip doctest #2 (schema)
+      When query
+        """
+        SELECT arrays_zip(vals1, vals2, vals3) AS zipped FROM VALUES (array(1L, 2L, 3L), array(2L, 4L, 6L), array(3L, 6L)) AS t(vals1, vals2, vals3)
+        """
+      Then query schema
+        """
+        root
+         |-- zipped: array (nullable = true)
+         |    |-- element: struct (containsNull = false)
+         |    |    |-- vals1: long (nullable = true)
+         |    |    |-- vals2: long (nullable = true)
+         |    |    |-- vals3: long (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/exists.feature b/python/pysail/tests/spark/function/features/array/exists.feature
new file mode 100644
index 000000000..fd5e7ffd4
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/exists.feature
@@ -0,0 +1,328 @@
+@function(lambda)
+Feature: exists higher-order function
+
+  Rule: Basic boolean predicate evaluation
+
+    Scenario Outline: Basic: 
+      When query
+        """
+        SELECT exists(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                | arr            | pred   | result |
+        | predicate matches at least one element returns true | array(1, 2, 3) | x > 2  | true   |
+        | predicate matches no elements returns false         | array(1, 2, 3) | x > 10 | false  |
+        | predicate matches all elements returns true         | array(1, 2, 3) | x > 0  | true   |
+        | single element array predicate true                 | array(5)       | x > 0  | true   |
+        | single element array predicate false                | array(5)       | x > 10 | false  |
+        | empty array always returns false                    | array()        | x > 0  | false  |
+
+  Rule: NULL array input
+
+    Scenario: typed NULL array input returns NULL
+      When query
+        """
+        SELECT exists(CAST(NULL AS ARRAY), x -> x > 0) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: untyped null literal is a type error
+      When query
+        """
+        SELECT exists(null, x -> x > 0) AS result
+        """
+      Then query error .*
+
+  Rule: NULL elements in array — three-valued logic
+
+    Scenario Outline: Three-valued logic: 
+      When query
+        """
+        SELECT exists(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                                         | arr                      | pred          | result |
+        | null in array when predicate returns false for null and true exists          | array(1, null, 3)        | x > 2         | true   |
+        | null in array when predicate returns true for some non-null element          | array(1, null, 3)        | x > 0         | true   |
+        | null in array when no non-null element matches and null makes predicate null | array(1, null, 3)        | x > 5         | NULL   |
+        | null element matched by IS NULL predicate                                    | array(1, null, 3)        | x IS NULL     | true   |
+        | IS NOT NULL predicate still true when non-null elements exist                | array(1, null, 3)        | x IS NOT NULL | true   |
+        | all null array with numeric predicate returns NULL                           | array(null, null)        | x > 0         | NULL   |
+        | all null array with IS NULL predicate returns true                           | array(null, null)        | x IS NULL     | true   |
+        | single typed null element matched by IS NULL                                 | array(CAST(NULL AS INT)) | x IS NULL     | true   |
+
+  Rule: Predicate returning NULL
+
+    Scenario Outline: Predicate returning NULL: 
+      When query
+        """
+        SELECT exists(array(1, 2, 3), x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                                       | pred                                                      | result |
+        | predicate always returns NULL results in NULL                              | CAST(NULL AS BOOLEAN)                                     | NULL   |
+        | predicate returns true for some elements and NULL for others returns true  | CASE WHEN x = 2 THEN true ELSE CAST(NULL AS BOOLEAN) END  | true   |
+        | predicate returns false for some elements and NULL for others returns NULL | CASE WHEN x = 2 THEN false ELSE CAST(NULL AS BOOLEAN) END | NULL   |
+
+  Rule: Lambda only accepts one parameter
+
+    Scenario: two-parameter lambda is rejected as type error
+      When query
+        """
+        SELECT exists(array(1, 2, 3), (x, i) -> x > i) AS result
+        """
+      Then query error .*
+
+  Rule: Element type coverage
+
+    Scenario Outline: Element type: 
+      When query
+        """
+        SELECT exists(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                            | arr                        | pred      | result |
+        | long array                      | array(1L, 2L, 3L)          | x > 2L    | true   |
+        | double array                    | array(1.0, 2.0, 3.0)       | x > 2.5   | true   |
+        | decimal array                   | array(1.5BD, 2.5BD, 3.5BD) | x > 3.0BD | true   |
+        | string array match found        | array('a', 'b', 'c')       | x = 'b'   | true   |
+        | string array no match           | array('a', 'b', 'c')       | x = 'z'   | false  |
+        | boolean array with true element | array(false, false, true)  | x         | true   |
+        | boolean array all false         | array(false, false, false) | x         | false  |
+
+  Rule: Complex predicates
+
+    Scenario Outline: Complex predicate: 
+      When query
+        """
+        SELECT exists(, ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                               | arr                                                               | lambda                     | result |
+        | AND predicate                      | array(1, 2, 3, 4, 5)                                              | x -> x > 2 AND x < 5       | true   |
+        | OR predicate                       | array(1, 2, 3)                                                    | x -> x < 0 OR x > 2        | true   |
+        | struct array field access          | array(named_struct('a', 1, 'b', 2), named_struct('a', 3, 'b', 4)) | s -> s.a > 2               | true   |
+        | struct array field access no match | array(named_struct('a', 1, 'b', 2), named_struct('a', 3, 'b', 4)) | s -> s.a > 10              | false  |
+        | nested array with inner exists     | array(array(1,2), array(3,4))                                     | x -> exists(x, y -> y > 3) | true   |
+
+  Rule: Outer column capture
+
+    Scenario: predicate references column from outer query
+      When query
+        """
+        SELECT exists(array(1, 2, 3), x -> x > v) AS result FROM (SELECT 2 AS v) t
+        """
+      Then query result
+        | result |
+        | true   |
+
+  Rule: ANSI mode inside the predicate
+
+    Scenario Outline: short-circuit avoids division by zero under ANSI 
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT exists(array(1, 0, 2), x -> 10 / x > 4) AS result
+        """
+      Then query result
+        | result |
+        | true   |
+
+      Examples:
+        | mode | ansi  |
+        | on   | true  |
+        | off  | false |
+
+  Rule: Predicate must return boolean
+
+    Scenario: non-boolean predicate is a type error
+      When query
+        """
+        SELECT exists(array(1, 2, 3), x -> x + 1) AS result
+        """
+      Then query error .*
+
+    @sail-bug
+    Scenario: a constant boolean is accepted in place of a lambda
+      When query
+        """
+        SELECT exists(array(1, 2), true) AS result
+        """
+      Then query result
+        | result |
+        | true   |
+
+  Rule: Array borne by a column rather than a literal
+
+    Scenario: distinct arrays per row are not broadcast from the first row
+      When query
+        """
+        SELECT exists(c, x -> x > 2) AS result
+        FROM VALUES (array(5)), (array(1)), (array(3)) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | true   |
+        | false  |
+        | true   |
+
+    Scenario: non-empty, empty and NULL arrays in the same batch
+      When query
+        """
+        SELECT exists(c, x -> x > 2) AS result
+        FROM VALUES (array(1, 2)), (array(3, 4)), (CAST(NULL AS ARRAY)), (array()) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | false  |
+        | true   |
+        | NULL   |
+        | false  |
+
+    Scenario: three-valued logic resolved per row
+      When query
+        """
+        SELECT exists(c, x -> x > 2) AS result
+        FROM VALUES (array(1, NULL)), (array(3, NULL)), (array(NULL)) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | NULL   |
+        | true   |
+        | NULL   |
+
+    Scenario Outline: every row is 
+      When query
+        """
+        SELECT exists(c, x -> x > 2) AS result
+        FROM VALUES  AS t(c)
+        """
+      Then query result ordered
+        | result   |
+        |  |
+        |  |
+
+      Examples:
+        | case           | values                                                 | result |
+        | a NULL array   | (CAST(NULL AS ARRAY)), (CAST(NULL AS ARRAY)) | NULL   |
+        | an empty array | (array()), (array())                                   | false  |
+
+    Scenario: the captured column changes the predicate per row
+      When query
+        """
+        SELECT exists(c, x -> x > v) AS result
+        FROM VALUES (array(1, 2), 0), (array(1, 2), 5) AS t(c, v)
+        """
+      Then query result ordered
+        | result |
+        | true   |
+        | false  |
+
+  Rule: Short-circuit order under ANSI
+
+    Scenario Outline:  under ANSI on
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT exists(, x -> ) AS result
+        """
+      Then query error Division by zero
+
+      Examples:
+        | case                                                | arr            | pred         |
+        | an element before the first true is still evaluated | array(0, 1, 2) | 10 / x > 4   |
+        | no element is true so every element is evaluated    | array(1, 0, 2) | 10 / x > 100 |
+        | a true after the failing element does not save it   | array(5, 0, 1) | 10 / x > 4   |
+
+    Scenario Outline:  under ANSI off
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT exists(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                | arr            | pred         | result |
+        | an element before the first true is still evaluated | array(0, 1, 2) | 10 / x > 4   | true   |
+        | no element is true so every element is evaluated    | array(1, 0, 2) | 10 / x > 100 | NULL   |
+        | a true after the failing element does not save it   | array(5, 0, 1) | 10 / x > 4   | true   |
+
+    Scenario Outline: one row short-circuits while another does not under ANSI 
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT exists(c, x -> 10 / x > 4) AS result
+        FROM VALUES (array(1, 0)), (array(5, 5)) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | true   |
+        | false  |
+
+      Examples:
+        | mode | ansi  |
+        | on   | true  |
+        | off  | false |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null array literal yields a non-nullable boolean
+      When query
+        """
+        SELECT exists(array(1, 2), x -> x > 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = false)
+        """
+
+    Scenario: a typed NULL array literal yields a nullable boolean
+      When query
+        """
+        SELECT exists(CAST(NULL AS ARRAY), x -> x > 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
+
+    Scenario: a nullable array column yields a nullable boolean
+      When query
+        """
+        SELECT exists(c, x -> x > 1) AS result
+        FROM VALUES (array(1)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/flatten.feature b/python/pysail/tests/spark/function/features/array/flatten.feature
new file mode 100644
index 000000000..87dd1823e
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/flatten.feature
@@ -0,0 +1,99 @@
+Feature: flatten() flattens nested arrays
+
+  Rule: Basic flattening
+
+    Scenario Outline: flatten nested array of 
+      When query
+        """
+        SELECT flatten() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case     | arr                                | result       |
+        | integers | array(array(1, 2), array(3, 4))    | [1, 2, 3, 4] |
+        | strings  | array(array('a', 'b'), array('c')) | [a, b, c]    |
+
+  Rule: Empty array edge cases
+
+    Scenario Outline: Empty array: 
+      When query
+        """
+        SELECT flatten() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                     | arr                                   | result    |
+        | flatten with empty inner arrays          | array(array(), array())               | []        |
+        | flatten single empty inner array         | array(array())                        | []        |
+        | flatten mixed empty and non-empty arrays | array(array(1), array(), array(2, 3)) | [1, 2, 3] |
+
+  Rule: Null handling
+
+    Scenario: flatten with null element in outer array
+      When query
+        """
+        SELECT flatten(array(array(1, 2), NULL, array(3))) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null nested array literal yields a non-nullable array
+      When query
+        """
+        SELECT flatten(array(array(1, 2), array(3))) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null nested array column yields a non-nullable array
+      When query
+        """
+        SELECT flatten(array(array(id))) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: long (containsNull = false)
+        """
+
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT flatten(c) AS result FROM VALUES (array(array(1))), (CAST(NULL AS ARRAY>)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
+
+    @sail-bug
+    Scenario: nullable inner elements propagate to the element nullability
+      When query
+        """
+        SELECT flatten(c) AS result FROM VALUES (array(array(1, CAST(NULL AS INT)))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/forall.feature b/python/pysail/tests/spark/function/features/array/forall.feature
new file mode 100644
index 000000000..9fdf56f0a
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/forall.feature
@@ -0,0 +1,336 @@
+@function(lambda)
+Feature: forall higher-order function
+
+  Rule: Basic boolean predicate evaluation
+
+    Scenario Outline: Basic: 
+      When query
+        """
+        SELECT forall(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                     | arr            | pred   | result |
+        | predicate true for all elements returns true             | array(1, 2, 3) | x > 0  | true   |
+        | predicate false for at least one element returns false   | array(1, 2, 3) | x > 1  | false  |
+        | predicate true for all when all values satisfy condition | array(2, 3, 4) | x > 1  | true   |
+        | single element predicate true                            | array(5)       | x > 0  | true   |
+        | single element predicate false                           | array(5)       | x > 10 | false  |
+        | empty array is vacuously true                            | array()        | x > 0  | true   |
+
+  Rule: NULL array input
+
+    Scenario: typed NULL array input returns NULL
+      When query
+        """
+        SELECT forall(CAST(NULL AS ARRAY), x -> x > 0) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  Rule: NULL elements in array — three-valued logic
+
+    Scenario Outline: Three-valued logic: 
+      When query
+        """
+        SELECT forall(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                                   | arr                      | pred          | result |
+        | null in array when some non-null element fails predicate returns false | array(1, null, 3)        | x > 2         | false  |
+        | null in array when all non-null elements pass predicate returns NULL   | array(2, null, 3)        | x > 1         | NULL   |
+        | null element with IS NOT NULL predicate returns false                  | array(1, null, 3)        | x IS NOT NULL | false  |
+        | null element with IS NULL predicate on mixed array returns false       | array(1, null, 3)        | x IS NULL     | false  |
+        | all null array with numeric predicate returns NULL                     | array(null, null)        | x > 0         | NULL   |
+        | all null array with IS NULL predicate returns true                     | array(null, null)        | x IS NULL     | true   |
+        | single typed null element with IS NULL predicate returns true          | array(CAST(NULL AS INT)) | x IS NULL     | true   |
+
+  Rule: Predicate returning NULL
+
+    Scenario Outline: Predicate returning NULL: 
+      When query
+        """
+        SELECT forall(array(1, 2, 3), x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                                      | pred                                                      | result |
+        | predicate always returns NULL results in NULL                             | CAST(NULL AS BOOLEAN)                                     | NULL   |
+        | predicate returns true for some elements and NULL for others returns NULL | CASE WHEN x = 2 THEN true ELSE CAST(NULL AS BOOLEAN) END  | NULL   |
+        | predicate returns false for any element overrides NULL and returns false  | CASE WHEN x = 2 THEN false ELSE CAST(NULL AS BOOLEAN) END | false  |
+
+  Rule: Lambda only accepts one parameter
+
+    Scenario: two-parameter lambda is rejected as type error
+      When query
+        """
+        SELECT forall(array(1, 2, 3), (x, i) -> x > i) AS result
+        """
+      Then query error .*
+
+  Rule: Element type coverage
+
+    Scenario Outline: Element type: 
+      When query
+        """
+        SELECT forall(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                   | arr                        | pred      | result |
+        | long array all satisfy condition       | array(1L, 2L, 3L)          | x > 0L    | true   |
+        | double array all satisfy condition     | array(1.0, 2.0, 3.0)       | x > 0.5   | true   |
+        | decimal array all satisfy condition    | array(1.5BD, 2.5BD, 3.5BD) | x > 0.0BD | true   |
+        | string array not all satisfy condition | array('a', 'b', 'c')       | x > 'a'   | false  |
+        | string array all satisfy condition     | array('b', 'c', 'd')       | x > 'a'   | true   |
+        | boolean array all true                 | array(true, true)          | x         | true   |
+        | boolean array contains false           | array(true, false)         | x         | false  |
+
+  Rule: Complex predicates
+
+    Scenario Outline: Complex predicate: 
+      When query
+        """
+        SELECT forall(, ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                     | arr                             | lambda                     | result |
+        | AND predicate all satisfy                | array(2, 4, 6)                  | x -> x > 0 AND x % 2 = 0   | true   |
+        | AND predicate not all satisfy            | array(2, 3, 6)                  | x -> x > 0 AND x % 2 = 0   | false  |
+        | nested array with inner forall all pass  | array(array(2, 4), array(6, 8)) | a -> forall(a, x -> x > 1) | true   |
+        | nested array with inner forall some fail | array(array(2, 4), array(0, 8)) | a -> forall(a, x -> x > 1) | false  |
+
+  Rule: Outer column capture
+
+    Scenario: predicate references column from outer query
+      When query
+        """
+        SELECT forall(array(1, 2, 3), x -> x > v) AS result FROM (SELECT 0 AS v) t
+        """
+      Then query result
+        | result |
+        | true   |
+
+  Rule: ANSI mode inside the predicate
+
+    Scenario: division by zero is reached without short-circuit under ANSI on
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT forall(array(1, 0, 2), x -> 10 / x > 4) AS result
+        """
+      Then query error .*
+
+    Scenario: division by zero yields a NULL predicate under ANSI off
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT forall(array(1, 0, 2), x -> 10 / x > 4) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  Rule: Predicate must return boolean
+
+    Scenario: non-boolean predicate is a type error
+      When query
+        """
+        SELECT forall(array(1, 2, 3), x -> x + 1) AS result
+        """
+      Then query error .*
+
+    @sail-bug
+    Scenario: a constant boolean is accepted in place of a lambda
+      When query
+        """
+        SELECT forall(array(1, 2), true) AS result
+        """
+      Then query result
+        | result |
+        | true   |
+
+  Rule: Array borne by a column rather than a literal
+
+    Scenario: distinct arrays per row are not broadcast from the first row
+      When query
+        """
+        SELECT forall(c, x -> x > 2) AS result
+        FROM VALUES (array(5)), (array(1)), (array(3)) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | true   |
+        | false  |
+        | true   |
+
+    Scenario: non-empty, empty and NULL arrays in the same batch
+      When query
+        """
+        SELECT forall(c, x -> x > 2) AS result
+        FROM VALUES (array(3, 4)), (array(1, 2)), (CAST(NULL AS ARRAY)), (array()) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | true   |
+        | false  |
+        | NULL   |
+        | true   |
+
+    Scenario: three-valued logic resolved per row
+      When query
+        """
+        SELECT forall(c, x -> x > 2) AS result
+        FROM VALUES (array(3, NULL)), (array(1, NULL)), (array(NULL)) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | NULL   |
+        | false  |
+        | NULL   |
+
+    Scenario Outline: every row is 
+      When query
+        """
+        SELECT forall(c, x -> x > 2) AS result
+        FROM VALUES  AS t(c)
+        """
+      Then query result ordered
+        | result   |
+        |  |
+        |  |
+
+      Examples:
+        | case           | values                                                 | result |
+        | a NULL array   | (CAST(NULL AS ARRAY)), (CAST(NULL AS ARRAY)) | NULL   |
+        | an empty array | (array()), (array())                                   | true   |
+
+    Scenario: the captured column changes the predicate per row
+      When query
+        """
+        SELECT forall(c, x -> x > v) AS result
+        FROM VALUES (array(1, 2), 0), (array(1, 2), 5) AS t(c, v)
+        """
+      Then query result ordered
+        | result |
+        | true   |
+        | false  |
+
+  Rule: Short-circuit order under ANSI
+
+    Scenario Outline: a false before the failing element stops evaluation under ANSI 
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT forall(array(100, 0, 2), x -> 10 / x > 4) AS result
+        """
+      Then query result
+        | result |
+        | false  |
+
+      Examples:
+        | mode | ansi  |
+        | on   | true  |
+        | off  | false |
+
+    Scenario Outline:  under ANSI on
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT forall(, x -> 10 / x > 4) AS result
+        """
+      Then query error Division by zero
+
+      Examples:
+        | case                                                    | arr              |
+        | the failing element comes first so it is evaluated      | array(0, 100)    |
+        | a false only after the failing element does not save it | array(1, 0, 100) |
+
+    Scenario Outline:  under ANSI off
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT forall(, x -> 10 / x > 4) AS result
+        """
+      Then query result
+        | result |
+        | false  |
+
+      Examples:
+        | case                                                    | arr              |
+        | the failing element comes first so it is evaluated      | array(0, 100)    |
+        | a false only after the failing element does not save it | array(1, 0, 100) |
+
+    Scenario Outline: one row stops early while another does not under ANSI 
+      Given config spark.sql.ansi.enabled = 
+      When query
+        """
+        SELECT forall(c, x -> 10 / x > 4) AS result
+        FROM VALUES (array(100, 0)), (array(1, 2)) AS t(c)
+        """
+      Then query result ordered
+        | result |
+        | false  |
+        | true   |
+
+      Examples:
+        | mode | ansi  |
+        | on   | true  |
+        | off  | false |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null array literal yields a non-nullable boolean
+      When query
+        """
+        SELECT forall(array(1, 2), x -> x > 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = false)
+        """
+
+    Scenario: a nullable array column yields a nullable boolean
+      When query
+        """
+        SELECT forall(c, x -> x > 1) AS result
+        FROM VALUES (array(1)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
+
+    @sail-bug
+    Scenario: a non-null array column yields a non-nullable boolean
+      When query
+        """
+        SELECT forall(array(id), x -> x > 0) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/get.feature b/python/pysail/tests/spark/function/features/array/get.feature
new file mode 100644
index 000000000..ed2e18bf6
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/get.feature
@@ -0,0 +1,65 @@
+Feature: get with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: get — the argument is resolved per row, not taken from the first row
+
+    @function(columnargs)
+    Scenario: get with the argument as a literal
+      When query
+        """
+        SELECT get(array(1, 2, 3), 0) AS result
+        """
+      Then query result ordered
+        | result |
+        | 1      |
+
+    # Sail applies the first row's value to every row: Sail returns ['1', '1'].
+    @function(columnargs) @sail-bug
+    Scenario: get takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT get(array(1, 2, 3), c) AS result FROM VALUES (1, 0), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result |
+        | 1      |
+        | NULL   |
+
+    @function(columnargs)
+    Scenario: get takes argument 2 from a column holding two different values
+      When query
+        """
+        SELECT get(array(10, 20, 30), c) AS result FROM VALUES (1, 0), (2, 2) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result |
+        | 10     |
+        | 30     |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to get yields the schema Spark declares
+      When query
+        """
+        SELECT get(array(1, 2, 3), 0) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to get stays nullable
+      When query
+        """
+        SELECT get(c, 0) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/sequence.feature b/python/pysail/tests/spark/function/features/array/sequence.feature
new file mode 100644
index 000000000..d529d0dd5
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/sequence.feature
@@ -0,0 +1,43 @@
+Feature: sequence output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to sequence yields the schema Spark declares
+      When query
+        """
+        SELECT sequence(1, 5) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null column input to sequence yields the schema Spark declares
+      When query
+        """
+        SELECT sequence(CAST(id AS INT), 5) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a nullable column input to sequence stays nullable
+      When query
+        """
+        SELECT sequence(c, 5) AS result FROM VALUES (1), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/shuffle.feature b/python/pysail/tests/spark/function/features/array/shuffle.feature
new file mode 100644
index 000000000..21dc2acb5
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/shuffle.feature
@@ -0,0 +1,28 @@
+Feature: shuffle output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to shuffle yields the schema Spark declares
+      When query
+        """
+        SELECT shuffle(array(1, 20, 3, 5)) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    Scenario: a nullable column input to shuffle stays nullable
+      When query
+        """
+        SELECT shuffle(c) AS result FROM VALUES (array(1, 20, 3, 5)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/slice.feature b/python/pysail/tests/spark/function/features/array/slice.feature
new file mode 100644
index 000000000..f00440926
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/slice.feature
@@ -0,0 +1,98 @@
+Feature: slice with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: slice — the argument may come from a column
+
+    @function(columnargs)
+    Scenario: slice with the argument as a literal
+      When query
+        """
+        SELECT slice(array(1, 2, 3, 4), 2, 2) AS result
+        """
+      Then query result ordered
+        | result |
+        | [2, 3] |
+
+    # Sail rejects the column: Sail errors: Invalid argument error: Non-nullable field of ListArray "item" cannot contain nulls
+    @function(columnargs) @sail-bug
+    Scenario Outline: slice takes argument  from a column containing NULL
+      When query
+        """
+        SELECT slice(array(1, 2, 3, 4), ) AS result FROM VALUES (1, 2), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result |
+        | [2, 3] |
+        | NULL   |
+
+      Examples:
+        | n | args |
+        | 2 | c, 2 |
+        | 3 | 2, c |
+
+    @function(columnargs)
+    Scenario: slice takes argument 2 from a column holding two different values
+      When query
+        """
+        SELECT slice(array(1, 2, 3, 4), c, 2) AS result FROM VALUES (1, 1), (2, 3) AS t(i, c) ORDER BY i
+        """
+      Then query result ordered
+        | result |
+        | [1, 2] |
+        | [3, 4] |
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null array literal yields a non-nullable array
+      When query
+        """
+        SELECT slice(array(1, 2, 3), 1, 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null array column yields a non-nullable array
+      When query
+        """
+        SELECT slice(array(id, id, id), 1, 2) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: long (containsNull = false)
+        """
+
+    Scenario: a nullable array column stays nullable
+      When query
+        """
+        SELECT slice(c, 1, 2) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: integer (containsNull = true)
+        """
+
+    @sail-bug
+    Scenario: nullable input elements propagate to the element nullability
+      When query
+        """
+        SELECT slice(c, 1, 2) AS result FROM VALUES (array(1, CAST(NULL AS INT))) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: integer (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/sort_array.feature b/python/pysail/tests/spark/function/features/array/sort_array.feature
new file mode 100644
index 000000000..aed87841f
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/sort_array.feature
@@ -0,0 +1,29 @@
+Feature: sort_array output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to sort_array yields the schema Spark declares
+      When query
+        """
+        SELECT sort_array(array('b', 'd', null, 'c', 'a'), true) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = true)
+        """
+
+    Scenario: a nullable column input to sort_array stays nullable
+      When query
+        """
+        SELECT sort_array(c, true) AS result FROM VALUES (array('b', 'd', null, 'c', 'a')), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: string (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/array/transform.feature b/python/pysail/tests/spark/function/features/array/transform.feature
new file mode 100644
index 000000000..ef95befcd
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/array/transform.feature
@@ -0,0 +1,400 @@
+@function(lambda)
+Feature: transform higher-order function
+
+  Rule: Basic 1-param transform — integer arithmetic
+
+    Scenario Outline: 1-param: 
+      When query
+        """
+        SELECT transform(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                    | arr                  | expr    | result          |
+        | Transform integers by multiplying by 2  | array(1, 2, 3)       | x * 2   | [2, 4, 6]       |
+        | Transform integers by adding a constant | array(10, 20, 30)    | x + 100 | [110, 120, 130] |
+        | Transform integers by negating          | array(1, 2, 3)       | -x      | [-1, -2, -3]    |
+        | Transform integers with modulo          | array(1, 2, 3, 4, 5) | x % 3   | [1, 2, 0, 1, 2] |
+        | Transform integers multiplied by zero   | array(5, 10, 15)     | x * 0   | [0, 0, 0]       |
+        | Transform single-element array          | array(42)            | x + 1   | [43]            |
+
+  Rule: Basic 2-param transform — element and index
+
+    Scenario Outline: 2-param: 
+      When query
+        """
+        SELECT transform(, (x, i) -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                           | arr                  | expr        | result            |
+        | Transform with index — add index to element                    | array(10, 20, 30)    | x + i       | [10, 21, 32]      |
+        | Transform with index — multiply element by index plus one      | array(10, 20, 30)    | x * (i + 1) | [10, 40, 90]      |
+        | Transform with index — return index only (0-based)             | array(100, 200, 300) | i           | [0, 1, 2]         |
+        | Transform with index from sequence — multiply element by index | sequence(1, 5)       | x * i       | [0, 2, 6, 12, 20] |
+        | Transform with index from sequence — return index only         | sequence(1, 3)       | i           | [0, 1, 2]         |
+
+  Rule: Type coercion — different output types
+
+    Scenario Outline: Type coercion: 
+      When query
+        """
+        SELECT transform(, x -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                            | arr                      | expr                           | result                     |
+        | Transform integers to strings via cast          | array(1, 2, 3)           | concat(cast(x as string), "s") | [1s, 2s, 3s]               |
+        | Transform integers to bigint                    | array(1, 2, 3)           | cast(x as bigint)              | [1, 2, 3]                  |
+        | Transform integers to double                    | array(1, 2, 3)           | cast(x as double)              | [1.0, 2.0, 3.0]            |
+        | Transform integers to booleans using comparison | array(1, 2, 3, 4)        | x > 2                          | [false, false, true, true] |
+        | Transform booleans by negation                  | array(true, false, true) | NOT x                          | [false, true, false]       |
+        | Transform booleans to integers                  | array(true, false, true) | cast(x as int)                 | [1, 0, 1]                  |
+
+  Rule: String transformations
+
+    Scenario Outline: String: 
+      When query
+        """
+        SELECT transform(arr, ) AS result
+        FROM VALUES () AS t(arr)
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                                | lambda                                 | values                   | result            |
+        | Transform string array to uppercase                                 | x -> upper(x)                          | array("hello", "world")  | [HELLO, WORLD]    |
+        | Transform string array to length of each string                     | x -> length(x)                         | array("a", "bb", "ccc")  | [1, 2, 3]         |
+        | Transform string array with index — concat element and index        | (x, i) -> concat(x, cast(i as string)) | array("a", "b", "c")     | [a0, b1, c2]      |
+        | Transform string array with index — concat longer strings and index | (x, i) -> concat(x, cast(i as string)) | array("apple", "banana") | [apple0, banana1] |
+
+    Scenario: Transform array containing an empty string preserves length
+      When query
+        """
+        SELECT size(transform(arr, x -> upper(x))) AS result
+        FROM VALUES (array("")) AS t(arr)
+        """
+      Then query result
+        | result |
+        | 1      |
+
+  Rule: Null handling
+
+    Scenario Outline: Null handling: 
+      When query
+        """
+        SELECT transform(, ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                                 | arr                         | lambda               | result         |
+        | Transform array containing null — null propagates through arithmetic | array(1, NULL, 3)           | x -> x * 2           | [2, NULL, 6]   |
+        | Transform array with null — null plus constant propagates            | array(1, NULL, 3)           | x -> x + 10          | [11, NULL, 13] |
+        | Transform null elements with coalesce — substitute null with default | array(1, NULL, 3)           | x -> coalesce(x, -1) | [1, -1, 3]     |
+        | Transform all-null array — all elements remain null                  | array(NULL, NULL)           | x -> x * 2           | [NULL, NULL]   |
+        | Transform single-null element array                                  | array(NULL)                 | x -> x + 1           | [NULL]         |
+        | Transform null array itself returns null                             | CAST(NULL AS ARRAY)    | x -> x * 2           | NULL           |
+        | Transform typed null array of strings returns null                   | CAST(NULL AS ARRAY) | x -> upper(x)        | NULL           |
+
+  Rule: Empty array
+
+    Scenario: Transform empty integer array returns empty array
+      When query
+        """
+        SELECT transform(array(), x -> x * 2) AS result
+        """
+      Then query result
+        | result |
+        | []     |
+
+  Rule: Outer column references in lambda
+
+    Scenario: Transform using outer column as addend
+      When query
+        """
+        SELECT transform(array(1, 2, 3), x -> x + id) AS result
+        FROM VALUES (10) AS t(id)
+        """
+      Then query result
+        | result       |
+        | [11, 12, 13] |
+
+    Scenario: Transform with varying outer column per row
+      When query
+        """
+        SELECT transform(arr, x -> x + offset) AS result
+        FROM VALUES (array(1, 2, 3), 10), (array(4, 5), 20) AS t(arr, offset)
+        """
+      Then query result
+        | result       |
+        | [11, 12, 13] |
+        | [24, 25]     |
+
+  Rule: Multi-row with mixed null arrays
+
+    Scenario: Transform multiple rows including null array row
+      When query
+        """
+        SELECT transform(a, x -> x + 1) AS result
+        FROM VALUES (array(1, 2, 3)), (array(10, 20)), (CAST(NULL AS ARRAY)) AS t(a)
+        """
+      Then query result
+        | result    |
+        | [2, 3, 4] |
+        | [11, 21]  |
+        | NULL      |
+
+    Scenario: Two-param transform across multiple rows with null array row
+      When query
+        """
+        SELECT transform(arr, (x, i) -> x + i) AS result
+        FROM VALUES (array(10, 20)), (CAST(NULL AS ARRAY)), (array(30, 40, 50)) AS t(arr)
+        """
+      Then query result
+        | result       |
+        | [10, 21]     |
+        | NULL         |
+        | [30, 41, 52] |
+
+  Rule: Arrays of arrays
+
+    Scenario Outline: Arrays of arrays: 
+      When query
+        """
+        SELECT transform(, ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                 | arr                             | lambda                                      | result                   |
+        | Transform array of arrays — return inner array size  | array(array(1, 2), array(3, 4)) | x -> size(x)                                | [2, 2]                   |
+        | Transform array of arrays — inner size plus constant | array(array(1, 2), array(3, 4)) | x -> size(x) + 10                           | [12, 12]                 |
+        | Nested transform — transform within transform        | array(1, 2, 3)                  | x -> transform(array(x, x + 1), y -> y * 2) | [[2, 4], [4, 6], [6, 8]] |
+
+  Rule: Struct output from lambda
+
+    Scenario: Transform to struct output
+      When query
+        """
+        SELECT transform(array(1, 2, 3), x -> struct(x, x * 2)) AS result
+        """
+      Then query result
+        | result                   |
+        | [{1, 2}, {2, 4}, {3, 6}] |
+
+    Scenario: Transform string array with index into struct of index and value
+      When query
+        """
+        SELECT transform(arr, (v, i) -> struct(i, v)) AS result
+        FROM VALUES (array("x", "y", "z")) AS t(arr)
+        """
+      Then query result
+        | result                   |
+        | [{0, x}, {1, y}, {2, z}] |
+
+  Rule: Chaining with other higher-order functions
+
+    Scenario: Transform then filter — double elements then keep those greater than 4
+      When query
+        """
+        SELECT filter(transform(array(1, 2, 3, 4), x -> x * 2), x -> x > 4) AS result
+        """
+      Then query result
+        | result |
+        | [6, 8] |
+
+  Rule: Large boundary values
+
+    Scenario: Transform INT_MAX and INT_MIN with identity
+      When query
+        """
+        SELECT transform(array(2147483647, -2147483648), x -> x + 0) AS result
+        """
+      Then query result
+        | result                    |
+        | [2147483647, -2147483648] |
+
+  Rule: Lambda body honors ANSI mode
+
+    # `transform` itself has no ANSI-specific semantics (Spark's ArrayTransform in
+    # higherOrderFunctions.scala has no ansiEnabled branch), but arithmetic inside
+    # the lambda body inherits the ANSI mode of the session.
+
+    # @sail-bug: NOT transform-specific. Sail's integer `+` does not honor
+    # `spark.sql.ansi.enabled` overflow checking — plain `SELECT 2147483647 + 1`
+    # also wraps to -2147483648 under ANSI on instead of erroring (CAST does
+    # error, arithmetic does not). The lambda body merely inherits this
+    # pre-existing general gap. Spark errors with ARITHMETIC_OVERFLOW.
+    @sail-bug
+    Scenario: Arithmetic overflow inside the lambda errors under ANSI on
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT transform(array(2147483647), x -> x + 1) AS result
+        """
+      Then query error (?i)overflow
+
+    Scenario: Arithmetic overflow inside the lambda wraps under ANSI off
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT transform(array(2147483647), x -> x + 1) AS result
+        """
+      Then query result
+        | result        |
+        | [-2147483648] |
+
+  Rule: Invalid arguments are rejected
+
+    Scenario Outline: Invalid argument: 
+      When query
+        """
+        SELECT transform() AS result
+        """
+      Then query error .*
+
+      Examples:
+        | case                                                  | args                           |
+        | Transform over a non-array first argument is rejected | 42, x -> x + 1                 |
+        | Transform over a map first argument is rejected       | map("a", 1), x -> x            |
+        | Lambda with three parameters is rejected              | array(1, 2, 3), (x, i, z) -> x |
+
+  Rule: Lambda body that ignores its parameters
+
+    Scenario: Constant lambda body broadcasts to every element
+      When query
+        """
+        SELECT transform(array(1, 2, 3), x -> 42) AS result
+        """
+      Then query result
+        | result       |
+        | [42, 42, 42] |
+
+  Rule: Index-only lambda used inside an expression (index-first rewrite)
+
+    # The element parameter is unused while the index is used in an expression.
+    # This exercises the planner's index-first rewrite — the lambda is rewritten
+    # to `i -> ...` over the index-first UDF instance — beyond the bare
+    # `(x, i) -> i` case covered above.
+
+    Scenario Outline: Index-first: 
+      When query
+        """
+        SELECT transform(array(10, 20, 30), (x, i) -> ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                           | expr    | result          |
+        | Index multiplied by a constant | i * 10  | [0, 10, 20]     |
+        | Index plus a constant          | i + 100 | [100, 101, 102] |
+
+  Rule: Two-parameter lambda over an empty array
+
+    Scenario: Empty array with element-and-index lambda returns empty array
+      When query
+        """
+        SELECT transform(array(), (x, i) -> x + i) AS result
+        """
+      Then query result
+        | result |
+        | []     |
+
+  Rule: Lambda returning a complex type
+
+    Scenario: Lambda returns an array — result is an array of arrays
+      When query
+        """
+        SELECT transform(array(1, 2, 3), x -> array(x, x * 2)) AS result
+        """
+      Then query result
+        | result                   |
+        | [[1, 2], [2, 4], [3, 6]] |
+
+  Rule: Lambda body produces conditional nulls
+
+    Scenario: IF expression yields NULL for some elements
+      When query
+        """
+        SELECT transform(array(1, 2, 3, 4), x -> if(x > 2, x, NULL)) AS result
+        """
+      Then query result
+        | result             |
+        | [NULL, NULL, 3, 4] |
+
+  Rule: Element referenced multiple times
+
+    Scenario: Square each element
+      When query
+        """
+        SELECT transform(array(1, 2, 3, 4), x -> x * x) AS result
+        """
+      Then query result
+        | result        |
+        | [1, 4, 9, 16] |
+
+  Rule: Nested transform capturing the outer index
+
+    Scenario: Inner lambda captures the outer 0-based index
+      When query
+        """
+        SELECT transform(array(1, 2), (x, i) -> transform(array(10, 20), (y, j) -> i * 100 + y + j)) AS result
+        """
+      Then query result
+        | result                 |
+        | [[10, 21], [110, 121]] |
+
+  Rule: Transform over an array of structs
+
+    Scenario: Project a struct field from each element
+      When query
+        """
+        SELECT transform(array(named_struct('k', 1, 'v', 10), named_struct('k', 2, 'v', 20)), x -> x.v) AS result
+        """
+      Then query result
+        | result   |
+        | [10, 20] |
+
+  Rule: Chaining — filter then transform
+
+    Scenario: Keep even elements then multiply by ten
+      When query
+        """
+        SELECT transform(filter(array(1, 2, 3, 4, 5), x -> x % 2 = 0), x -> x * 10) AS result
+        """
+      Then query result
+        | result   |
+        | [20, 40] |
+
+  Rule: Decimal element arithmetic
+
+    # `decimal(2,1) * int` widens to `decimal(4,1)` in Spark, so the result
+    # renders with the `.0` suffix. If Sail diverges here it is the pre-existing
+    # decimal×int coercion gap, not a `transform` issue.
+
+    Scenario: Multiply decimal elements by an integer
+      When query
+        """
+        SELECT transform(array(1.5, 2.5, 3.5), x -> x * 2) AS result
+        """
+      Then query result
+        | result          |
+        | [3.0, 5.0, 7.0] |
diff --git a/python/pysail/tests/spark/function/features/array_coercion.feature b/python/pysail/tests/spark/function/features/array_coercion.feature
deleted file mode 100644
index 225b2d561..000000000
--- a/python/pysail/tests/spark/function/features/array_coercion.feature
+++ /dev/null
@@ -1,85 +0,0 @@
-Feature: array() type coercion with mixed element types
-
-  Spark's non-ANSI semantics coerce mixed string/numeric arrays to string,
-  not to numeric. DataFusion 54 changed `comparison_coercion` to prefer
-  numeric, which would otherwise reject `array('a', 1)` at runtime.
-
-  Rule: Homogeneous arrays preserve their element type
-
-    Scenario: array of integers
-      When query
-      """
-      SELECT array(1, 2, 3) AS result
-      """
-      Then query result
-      | result    |
-      | [1, 2, 3] |
-
-    Scenario: array of strings
-      When query
-      """
-      SELECT array('a', 'b', 'c') AS result
-      """
-      Then query result
-      | result    |
-      | [a, b, c] |
-
-    Scenario: array of doubles
-      When query
-      """
-      SELECT array(1.0, 2.5, 3.5) AS result
-      """
-      Then query result
-      | result          |
-      | [1.0, 2.5, 3.5] |
-
-  Rule: Mixed numeric types coerce to the widest numeric type
-
-    Scenario: integer and double coerce to double
-      When query
-      """
-      SELECT array(1, 2.5) AS result
-      """
-      Then query result
-      | result     |
-      | [1.0, 2.5] |
-
-  Rule: Mixed string and numeric types coerce to string (Spark non-ANSI)
-
-    Scenario: string and integer coerce to string
-      When query
-      """
-      SELECT array('a', 1) AS result
-      """
-      Then query result
-      | result |
-      | [a, 1] |
-
-    Scenario: string and double coerce to string
-      When query
-      """
-      SELECT array('a', 1.5) AS result
-      """
-      Then query result
-      | result    |
-      | [a, 1.5] |
-
-    Scenario: multiple strings and numerics coerce to string
-      When query
-      """
-      SELECT array('a', 1, 2.5, 'b') AS result
-      """
-      Then query result
-      | result          |
-      | [a, 1, 2.5, b]  |
-
-  Rule: NULL elements are preserved during coercion
-
-    Scenario: string, numeric and NULL coerce to string with NULL preserved
-      When query
-      """
-      SELECT array('a', 1, NULL, 1.0) AS result
-      """
-      Then query result
-      | result              |
-      | [a, 1, NULL, 1.0]   |
diff --git a/python/pysail/tests/spark/function/features/array_compact.feature b/python/pysail/tests/spark/function/features/array_compact.feature
deleted file mode 100644
index c9db3399e..000000000
--- a/python/pysail/tests/spark/function/features/array_compact.feature
+++ /dev/null
@@ -1,129 +0,0 @@
-Feature: array_compact() removes null values from an array
-
-  Rule: Basic usage
-
-    Scenario: array_compact removes null values from integer array
-      When query
-        """
-        SELECT array_compact(array(1, NULL, 2, NULL, 3)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-
-    Scenario: array_compact removes null values from string array
-      When query
-        """
-        SELECT array_compact(array('a', NULL, 'b', NULL, 'c')) AS result
-        """
-      Then query result
-        | result    |
-        | [a, b, c] |
-
-    Scenario: array_compact with no null values returns same array
-      When query
-        """
-        SELECT array_compact(array(1, 2, 3)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-
-    Scenario: array_compact with all null values returns empty array
-      When query
-        """
-        SELECT array_compact(array(CAST(NULL AS INT), CAST(NULL AS INT))) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: array_compact with null at beginning
-      When query
-        """
-        SELECT array_compact(array(NULL, 1, 2, 3)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-
-    Scenario: array_compact with null at end
-      When query
-        """
-        SELECT array_compact(array(1, 2, 3, NULL)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-
-  Rule: Empty array handling
-
-    Scenario: array_compact with empty array returns empty array
-      When query
-        """
-        SELECT array_compact(array()) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-  Rule: Null input propagation
-
-    Scenario: array_compact with null input returns null
-      When query
-        """
-        SELECT array_compact(NULL) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: array_compact with typed null input returns null
-      When query
-        """
-        SELECT array_compact(CAST(NULL AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Multiple rows
-
-    Scenario: array_compact across multiple rows
-      When query
-        """
-        SELECT id, array_compact(arr) AS result
-        FROM VALUES
-          (1, array(1, NULL, 2)),
-          (2, array(NULL, NULL)),
-          (3, array(3, 4, 5)),
-          (4, CAST(NULL AS ARRAY))
-        AS t(id, arr)
-        ORDER BY id
-        """
-      Then query result ordered
-        | id | result    |
-        | 1  | [1, 2]    |
-        | 2  | []        |
-        | 3  | [3, 4, 5] |
-        | 4  | NULL      |
-
-  Rule: Various data types
-
-    Scenario: array_compact with double values
-      When query
-        """
-        SELECT array_compact(array(1.1, NULL, 2.2, NULL)) AS result
-        """
-      Then query result
-        | result     |
-        | [1.1, 2.2] |
-
-    Scenario: array_compact with boolean values
-      When query
-        """
-        SELECT array_compact(array(true, NULL, false)) AS result
-        """
-      Then query result
-        | result       |
-        | [true, false] |
diff --git a/python/pysail/tests/spark/function/features/array_concat.feature b/python/pysail/tests/spark/function/features/array_concat.feature
deleted file mode 100644
index 1a53f744d..000000000
--- a/python/pysail/tests/spark/function/features/array_concat.feature
+++ /dev/null
@@ -1,71 +0,0 @@
-@sail-only
-Feature: array_concat() concatenates arrays (Sail extension)
-
-  Note: array_concat is a Sail extension not available in standard Spark.
-  Use concat() for Spark-compatible array concatenation.
-
-  Rule: Basic concatenation
-
-    @sail-only
-    Scenario: array_concat two integer arrays
-      When query
-      """
-      SELECT array_concat(array(1, 2, 3), array(4, 5)) AS result
-      """
-      Then query result
-      | result          |
-      | [1, 2, 3, 4, 5] |
-
-    @sail-only
-    Scenario: array_concat two string arrays
-      When query
-      """
-      SELECT array_concat(array('a', 'b'), array('c')) AS result
-      """
-      Then query result
-      | result    |
-      | [a, b, c] |
-
-  Rule: Empty array handling
-
-    @sail-only
-    Scenario: array_concat empty array with typed array
-      When query
-      """
-      SELECT array_concat(array(), array(1, 2, 3)) AS result
-      """
-      Then query result
-      | result    |
-      | [1, 2, 3] |
-
-    @sail-only
-    Scenario: array_concat typed array with empty array
-      When query
-      """
-      SELECT array_concat(array(1, 2), array()) AS result
-      """
-      Then query result
-      | result |
-      | [1, 2] |
-
-  Rule: Null propagation
-
-    @sail-only
-    Scenario: array_concat array with null returns null
-      When query
-      """
-      SELECT array_concat(array(1, 2), CAST(NULL AS ARRAY)) AS result
-      """
-      Then query result
-      | result |
-      | NULL   |
-
-    @sail-only
-    Scenario: array_concat null with array returns null
-      When query
-      """
-      SELECT array_concat(CAST(NULL AS ARRAY), array(1, 2)) AS result
-      """
-      Then query result
-      | result |
-      | NULL   |
diff --git a/python/pysail/tests/spark/function/features/array_intersect.feature b/python/pysail/tests/spark/function/features/array_intersect.feature
deleted file mode 100644
index 0a8893916..000000000
--- a/python/pysail/tests/spark/function/features/array_intersect.feature
+++ /dev/null
@@ -1,196 +0,0 @@
-Feature: array_intersect() returns common array elements without duplicates
-
-  Rule: Source-of-truth examples
-
-    Scenario: array_intersect basic usage
-      When query
-        """
-        SELECT sort_array(array_intersect(array('b', 'a', 'c'), array('c', 'd', 'a', 'f'))) AS result
-        """
-      Then query result
-        | result |
-        | [a, c] |
-
-    Scenario: array_intersect on array columns preserves first-array order
-      When query
-        """
-        SELECT array_intersect(c1, c2) AS result
-        FROM VALUES
-          (array('b', 'a', 'c'), array('c', 'd', 'a', 'f'))
-        AS t(c1, c2)
-        """
-      Then query result
-        | result |
-        | [a, c] |
-
-    Scenario: array_intersect preserves left-array order when left is shorter
-      When query
-        """
-        SELECT array_intersect(c1, c2) AS result
-        FROM VALUES
-          (array(1, 2), array(2, 1, 3))
-        AS t(c1, c2)
-        """
-      Then query result
-        | result |
-        | [1, 2] |
-
-    Scenario: array_intersect with no common elements
-      When query
-        """
-        SELECT array_intersect(array('b', 'a', 'c'), array('d', 'e', 'f')) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: array_intersect with all common elements
-      When query
-        """
-        SELECT sort_array(array_intersect(array('a', 'b', 'c'), array('a', 'b', 'c'))) AS result
-        """
-      Then query result
-        | result    |
-        | [a, b, c] |
-
-    Scenario: array_intersect with null values
-      When query
-        """
-        SELECT sort_array(array_intersect(array('a', 'b', NULL), array('a', NULL, 'c'))) AS result
-        """
-      Then query result
-        | result    |
-        | [NULL, a] |
-
-    Scenario: array_intersect with empty arrays
-      When query
-        """
-        SELECT array_intersect(CAST(array() AS ARRAY), array('a', 'b', 'c')) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-  Rule: Duplicates and multiple rows
-
-    Scenario: array_intersect removes duplicates from the result
-      When query
-        """
-        SELECT sort_array(array_intersect(array(1, 2, 2, 3, 3), array(2, 2, 3, 4))) AS result
-        """
-      Then query result
-        | result |
-        | [2, 3] |
-
-    Scenario: array_intersect on identical arrays deduplicates in left-array order
-      When query
-        """
-        SELECT array_intersect(c1, c2) AS result
-        FROM VALUES
-          (array('b', 'a', 'c', 'c'), array('b', 'a', 'c', 'c'))
-        AS t(c1, c2)
-        """
-      Then query result
-        | result  |
-        | [b, a, c] |
-
-    Scenario: array_intersect across multiple rows
-      When query
-        """
-        SELECT id, sort_array(array_intersect(left_arr, right_arr)) AS result
-        FROM VALUES
-          (1, array(1, 2, 3), array(2, 3, 4)),
-          (2, array(1, 1, 1), array(1, 2)),
-          (3, array(1, 2), array(3, 4)),
-          (4, CAST(NULL AS ARRAY), array(1, 2))
-        AS t(id, left_arr, right_arr)
-        ORDER BY id
-        """
-      Then query result ordered
-        | id | result |
-        | 1  | [2, 3] |
-        | 2  | [1]    |
-        | 3  | []     |
-        | 4  | NULL   |
-
-  Rule: Type-specific behavior
-
-    Scenario: array_intersect with boolean arrays
-      When query
-        """
-        SELECT sort_array(array_intersect(array(true, false, true), array(false, false))) AS result
-        """
-      Then query result
-        | result  |
-        | [false] |
-
-    Scenario: array_intersect with double arrays
-      When query
-        """
-        SELECT sort_array(array_intersect(array(1.5D, 2.5D, 3.5D), array(3.5D, 4.5D, 1.5D))) AS result
-        """
-      Then query result
-        | result     |
-        | [1.5, 3.5] |
-
-    Scenario: array_intersect with arrays of structs
-      When query
-        """
-        SELECT array_intersect(
-          array(named_struct('id', 1, 'name', 'a'), named_struct('id', 2, 'name', 'b')),
-          array(named_struct('id', 2, 'name', 'b'), named_struct('id', 3, 'name', 'c'))
-        ) AS result
-        """
-      Then query result
-        | result   |
-        | [{2, b}] |
-
-    Scenario: array_intersect with array element type
-      When query
-        """
-        SELECT id, array_intersect(left_arr, right_arr) AS result
-        FROM VALUES
-          (1, array(NULL), array(NULL, NULL)),
-          (2, array(NULL), array()),
-          (3, array(NULL, NULL), array(NULL))
-        AS t(id, left_arr, right_arr)
-        ORDER BY id
-        """
-      Then query result ordered
-        | id | result |
-        | 1  | [NULL] |
-        | 2  | []     |
-        | 3  | [NULL] |
-
-    Scenario: array elements do not match typed non-null elements
-      When query
-        """
-        SELECT id, array_intersect(left_arr, right_arr) AS result
-        FROM VALUES
-          (1, array(NULL, NULL), array('d', 'e', 'f')),
-          (2, array(NULL),       CAST(array() AS ARRAY)),
-          (3, array(),           CAST(array() AS ARRAY))
-        AS t(id, left_arr, right_arr)
-        ORDER BY id
-        """
-      Then query result ordered
-        | id | result |
-        | 1  | []     |
-        | 2  | []     |
-        | 3  | []     |
-
-  Rule: Invalid inputs
-
-    Scenario: array_intersect rejects non-array inputs
-      When query
-        """
-        SELECT array_intersect(1, array(1)) AS result
-        """
-      Then query error .*
-
-    Scenario: array_intersect rejects incompatible element types
-      When query
-        """
-        SELECT array_intersect(array(1), array('1')) AS result
-        """
-      Then query error .*
diff --git a/python/pysail/tests/spark/function/features/array_min_max.feature b/python/pysail/tests/spark/function/features/array_min_max.feature
deleted file mode 100644
index 4b826bdeb..000000000
--- a/python/pysail/tests/spark/function/features/array_min_max.feature
+++ /dev/null
@@ -1,390 +0,0 @@
-Feature: array_min and array_max functions
-
-  Rule: Basic usage
-
-    Scenario: array_min and array_max with integers
-      When query
-        """
-        SELECT array_min(array(3, 1, 2)) AS min_val, array_max(array(3, 1, 2)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1       | 3       |
-
-    Scenario: array_min and array_max with strings
-      When query
-        """
-        SELECT array_min(array('banana', 'apple', 'cherry')) AS min_val, array_max(array('banana', 'apple', 'cherry')) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | apple   | cherry  |
-
-    Scenario: array_min and array_max with doubles
-      When query
-        """
-        SELECT array_min(array(3.14, 1.5, 2.7)) AS min_val, array_max(array(3.14, 1.5, 2.7)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1.50    | 3.14    |
-
-    Scenario: array_min and array_max with single element
-      When query
-        """
-        SELECT array_min(array(42)) AS min_val, array_max(array(42)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 42      | 42      |
-
-    Scenario: array_min and array_max with repeated elements
-      When query
-        """
-        SELECT array_min(array(5, 5, 5)) AS min_val, array_max(array(5, 5, 5)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 5       | 5       |
-
-  Rule: Empty and NULL inputs
-
-    Scenario: array_min and array_max with empty array
-      When query
-        """
-        SELECT array_min(array()) AS min_val, array_max(array()) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | NULL    | NULL    |
-
-    Scenario: array_min and array_max with NULL input
-      When query
-        """
-        SELECT array_min(NULL) AS min_val, array_max(NULL) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | NULL    | NULL    |
-
-    Scenario: array_min and array_max with all NULLs
-      When query
-        """
-        SELECT array_min(array(CAST(NULL AS INT), CAST(NULL AS INT))) AS min_val, array_max(array(CAST(NULL AS INT), CAST(NULL AS INT))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | NULL    | NULL    |
-
-    Scenario: array_min and array_max with some NULLs mixed
-      When query
-        """
-        SELECT array_min(array(3, NULL, 1, NULL, 2)) AS min_val, array_max(array(3, NULL, 1, NULL, 2)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1       | 3       |
-
-    Scenario: array_min and array_max with NULL at first position
-      When query
-        """
-        SELECT array_min(array(NULL, 2, 3)) AS min_val, array_max(array(NULL, 2, 3)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 2       | 3       |
-
-    Scenario: array_min and array_max with NULL at last position
-      When query
-        """
-        SELECT array_min(array(1, 2, NULL)) AS min_val, array_max(array(1, 2, NULL)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1       | 2       |
-
-  Rule: Negative numbers
-
-    Scenario: array_min and array_max with negative numbers
-      When query
-        """
-        SELECT array_min(array(-5, -1, -10)) AS min_val, array_max(array(-5, -1, -10)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | -10     | -1      |
-
-    Scenario: array_min and array_max with mixed positive negative and zero
-      When query
-        """
-        SELECT array_min(array(-3, 0, 3)) AS min_val, array_max(array(-3, 0, 3)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | -3      | 3       |
-
-  Rule: Float special values
-
-    Scenario: array_min and array_max with NaN
-      When query
-        """
-        SELECT array_min(array(1.0, CAST('NaN' AS DOUBLE), 2.0)) AS min_val, array_max(array(1.0, CAST('NaN' AS DOUBLE), 2.0)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1.0     | NaN     |
-
-    Scenario: array_min and array_max with NaN only
-      When query
-        """
-        SELECT array_min(array(CAST('NaN' AS DOUBLE))) AS min_val, array_max(array(CAST('NaN' AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | NaN     | NaN     |
-
-    Scenario: array_min and array_max with NaN and NULL
-      When query
-        """
-        SELECT array_min(array(CAST('NaN' AS DOUBLE), NULL, 1.0)) AS min_val, array_max(array(CAST('NaN' AS DOUBLE), NULL, 1.0)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1.0     | NaN     |
-
-    Scenario: array_min and array_max with NaN vs Infinity
-      When query
-        """
-        SELECT array_min(array(CAST('NaN' AS DOUBLE), CAST('Infinity' AS DOUBLE))) AS min_val, array_max(array(CAST('NaN' AS DOUBLE), CAST('Infinity' AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val  | max_val |
-        | Infinity | NaN     |
-
-    Scenario: array_min and array_max with NaN vs negative Infinity
-      When query
-        """
-        SELECT array_min(array(CAST('-Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE))) AS min_val, array_max(array(CAST('-Infinity' AS DOUBLE), CAST('NaN' AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val   | max_val |
-        | -Infinity | NaN     |
-
-    Scenario: array_min and array_max with positive Infinity
-      When query
-        """
-        SELECT array_min(array(1.0, CAST('Infinity' AS DOUBLE))) AS min_val, array_max(array(1.0, CAST('Infinity' AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val | max_val  |
-        | 1.0     | Infinity |
-
-    Scenario: array_min and array_max with negative Infinity
-      When query
-        """
-        SELECT array_min(array(1.0, CAST('-Infinity' AS DOUBLE))) AS min_val, array_max(array(1.0, CAST('-Infinity' AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val   | max_val |
-        | -Infinity | 1.0     |
-
-    Scenario: array_min and array_max with both Infinities
-      When query
-        """
-        SELECT array_min(array(CAST('-Infinity' AS DOUBLE), CAST('Infinity' AS DOUBLE))) AS min_val, array_max(array(CAST('-Infinity' AS DOUBLE), CAST('Infinity' AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val   | max_val  |
-        | -Infinity | Infinity |
-
-    Scenario: array_min and array_max with float NaN
-      When query
-        """
-        SELECT array_min(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT))) AS min_val, array_max(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1.0     | NaN     |
-
-    Scenario: array_min and array_max with positive and negative zero
-      When query
-        """
-        SELECT array_min(array(CAST(0.0 AS DOUBLE), CAST(-0.0 AS DOUBLE))) AS min_val, array_max(array(CAST(0.0 AS DOUBLE), CAST(-0.0 AS DOUBLE))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 0.0     | 0.0     |
-
-    @sail-only
-    Scenario: array_min and array_max with extreme double values (display format differs from Spark)
-      When query
-        """
-        SELECT array_min(array(1.7976931348623157E308, -1.7976931348623157E308, 0.0)) AS min_val, array_max(array(1.7976931348623157E308, -1.7976931348623157E308, 0.0)) AS max_val
-        """
-      Then query result
-        | min_val                 | max_val                |
-        | -1.7976931348623157e308 | 1.7976931348623157e308 |
-
-  Rule: Boolean arrays
-
-    Scenario: array_min and array_max with booleans
-      When query
-        """
-        SELECT array_min(array(true, false)) AS min_val, array_max(array(true, false)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | false   | true    |
-
-  Rule: String edge cases
-
-    Scenario: array_min and array_max with empty string in array
-      When query
-        """
-        SELECT array_min(array('', 'a', 'b')) AS min_val, array_max(array('', 'a', 'b')) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        |         | b       |
-
-    Scenario: array_min and array_max with case sensitive strings
-      When query
-        """
-        SELECT array_min(array('Z', 'a', 'A')) AS min_val, array_max(array('Z', 'a', 'A')) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | A       | a       |
-
-    Scenario: array_min and array_max with mixed case and numeric strings
-      When query
-        """
-        SELECT array_min(array('abc', 'ABc', '123')) AS min_val, array_max(array('abc', 'ABc', '123')) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 123     | abc     |
-
-  Rule: Date and timestamp arrays
-
-    Scenario: array_min and array_max with dates
-      When query
-        """
-        SELECT array_min(array(DATE '2023-01-01', DATE '2023-12-31', DATE '2023-06-15')) AS min_val, array_max(array(DATE '2023-01-01', DATE '2023-12-31', DATE '2023-06-15')) AS max_val
-        """
-      Then query result
-        | min_val    | max_val    |
-        | 2023-01-01 | 2023-12-31 |
-
-    Scenario: array_min and array_max with timestamps
-      When query
-        """
-        SELECT array_min(array(TIMESTAMP '2023-01-01 00:00:00', TIMESTAMP '2023-12-31 23:59:59', TIMESTAMP '2023-06-15 12:00:00')) AS min_val, array_max(array(TIMESTAMP '2023-01-01 00:00:00', TIMESTAMP '2023-12-31 23:59:59', TIMESTAMP '2023-06-15 12:00:00')) AS max_val
-        """
-      Then query result
-        | min_val             | max_val             |
-        | 2023-01-01 00:00:00 | 2023-12-31 23:59:59 |
-
-    Scenario: array_min and array_max with dates and NULL
-      When query
-        """
-        SELECT array_min(array(DATE '2023-01-01', NULL, DATE '2023-12-31')) AS min_val, array_max(array(DATE '2023-01-01', NULL, DATE '2023-12-31')) AS max_val
-        """
-      Then query result
-        | min_val    | max_val    |
-        | 2023-01-01 | 2023-12-31 |
-
-  Rule: Decimal arrays
-
-    Scenario: array_min and array_max with decimals
-      When query
-        """
-        SELECT array_min(array(CAST(1.11 AS DECIMAL(10,2)), CAST(2.22 AS DECIMAL(10,2)), CAST(0.99 AS DECIMAL(10,2)))) AS min_val, array_max(array(CAST(1.11 AS DECIMAL(10,2)), CAST(2.22 AS DECIMAL(10,2)), CAST(0.99 AS DECIMAL(10,2)))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 0.99    | 2.22    |
-
-    Scenario: array_min with equal decimal values of different precision
-      When query
-        """
-        SELECT array_min(array(CAST(0.1 AS DECIMAL(38,18)), CAST(0.10 AS DECIMAL(38,18)), CAST(0.100 AS DECIMAL(38,18)))) AS min_val
-        """
-      Then query result
-        | min_val              |
-        | 0.100000000000000000 |
-
-  Rule: BIGINT boundary values
-
-    Scenario: array_min and array_max with BIGINT extremes
-      When query
-        """
-        SELECT array_min(array(CAST(9223372036854775807 AS BIGINT), CAST(-9223372036854775808 AS BIGINT), CAST(0 AS BIGINT))) AS min_val, array_max(array(CAST(9223372036854775807 AS BIGINT), CAST(-9223372036854775808 AS BIGINT), CAST(0 AS BIGINT))) AS max_val
-        """
-      Then query result
-        | min_val              | max_val             |
-        | -9223372036854775808 | 9223372036854775807 |
-
-    Scenario: array_min and array_max with SMALLINT extremes
-      When query
-        """
-        SELECT array_min(array(CAST(1 AS SMALLINT), CAST(-32768 AS SMALLINT), CAST(32767 AS SMALLINT))) AS min_val, array_max(array(CAST(1 AS SMALLINT), CAST(-32768 AS SMALLINT), CAST(32767 AS SMALLINT))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | -32768  | 32767   |
-
-    Scenario: array_min and array_max with TINYINT extremes
-      When query
-        """
-        SELECT array_min(array(CAST(1 AS TINYINT), CAST(-128 AS TINYINT), CAST(127 AS TINYINT))) AS min_val, array_max(array(CAST(1 AS TINYINT), CAST(-128 AS TINYINT), CAST(127 AS TINYINT))) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | -128    | 127     |
-
-  Rule: Large arrays
-
-    Scenario: array_min and array_max with large sequence
-      When query
-        """
-        SELECT array_min(sequence(1, 1000)) AS min_val, array_max(sequence(1, 1000)) AS max_val
-        """
-      Then query result
-        | min_val | max_val |
-        | 1       | 1000    |
-
-  Rule: Nested arrays
-
-    Scenario: array_min with nested arrays
-      When query
-        """
-        SELECT array_min(array(array(1,2), array(3,4))) AS min_val
-        """
-      Then query result
-        | min_val |
-        | [1, 2]  |
-
-    Scenario: array_max with nested arrays
-      When query
-        """
-        SELECT array_max(array(array(1,2), array(3,4))) AS max_val
-        """
-      Then query result
-        | max_val |
-        | [3, 4]  |
-
-  Rule: Multi-row results
-
-    Scenario: array_min and array_max across multiple rows
-      When query
-        """
-        SELECT id, array_min(arr) AS min_val, array_max(arr) AS max_val
-        FROM VALUES (1, array(10, 20, 30)), (2, array(5, 15, 25)), (3, array(100)) AS t(id, arr)
-        ORDER BY id
-        """
-      Then query result ordered
-        | id | min_val | max_val |
-        | 1  | 10      | 30      |
-        | 2  | 5       | 25      |
-        | 3  | 100     | 100     |
diff --git a/python/pysail/tests/spark/function/features/array_sort.feature b/python/pysail/tests/spark/function/features/array_sort.feature
deleted file mode 100644
index fe1f20418..000000000
--- a/python/pysail/tests/spark/function/features/array_sort.feature
+++ /dev/null
@@ -1,187 +0,0 @@
-@array_sort
-Feature: array_sort higher-order function
-
-  Rule: No-comparator form — natural ascending order, NULLs last
-
-    Scenario: Sort integers ascending
-      When query
-        """
-        SELECT array_sort(array(3, 1, 2)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-
-    Scenario: Sort integers with a null — null placed last
-      When query
-        """
-        SELECT array_sort(array(3, NULL, 1)) AS result
-        """
-      Then query result
-        | result          |
-        | [1, 3, NULL]    |
-
-    Scenario: Sort strings ascending
-      When query
-        """
-        SELECT array_sort(array('b', 'a', 'c')) AS result
-        """
-      Then query result
-        | result          |
-        | [a, b, c]       |
-
-    Scenario: Sort doubles with NaN and Infinity — NaN is greatest
-      When query
-        """
-        SELECT array_sort(array(double('NaN'), 1.0, double('Infinity'), -1.0)) AS result
-        """
-      Then query result
-        | result                     |
-        | [-1.0, 1.0, Infinity, NaN] |
-
-    Scenario: Sort null array returns null
-      When query
-        """
-        SELECT array_sort(CAST(NULL AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Comparator form — ascending and descending
-
-    Scenario: Ascending comparator with case expression
-      When query
-        """
-        SELECT array_sort(array(5, 6, 1), (l, r) -> CASE WHEN l < r THEN -1 WHEN l > r THEN 1 ELSE 0 END) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 5, 6] |
-
-    Scenario: Ascending comparator with subtraction
-      When query
-        """
-        SELECT array_sort(array(5, 6, 1), (l, r) -> l - r) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 5, 6] |
-
-    Scenario: Descending comparator with subtraction
-      When query
-        """
-        SELECT array_sort(array(5, 6, 1), (l, r) -> r - l) AS result
-        """
-      Then query result
-        | result    |
-        | [6, 5, 1] |
-
-  Rule: Comparator form — element types
-
-    Scenario: Comparator over array of structs by field
-      When query
-        """
-        SELECT array_sort(array(named_struct('x', 3), named_struct('x', 1), named_struct('x', 2)), (l, r) -> l.x - r.x) AS result
-        """
-      Then query result
-        | result                  |
-        | [{1}, {2}, {3}]         |
-
-    Scenario: Comparator over strings with explicit null handling — nulls first
-      When query
-        """
-        SELECT array_sort(array('bc', 'ab', NULL, 'dc'), (l, r) -> CASE WHEN l IS NULL AND r IS NULL THEN 0 WHEN l IS NULL THEN -1 WHEN r IS NULL THEN 1 WHEN l < r THEN 1 WHEN l > r THEN -1 ELSE 0 END) AS result
-        """
-      Then query result
-        | result                |
-        | [NULL, dc, bc, ab]    |
-
-  Rule: Comparator form — degenerate inputs
-
-    Scenario: Comparator on single-element array
-      When query
-        """
-        SELECT array_sort(array(5), (l, r) -> l - r) AS result
-        """
-      Then query result
-        | result |
-        | [5]    |
-
-    Scenario: Comparator on empty array
-      When query
-        """
-        SELECT array_sort(CAST(array() AS ARRAY), (l, r) -> l - r) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: Comparator on null array returns null
-      When query
-        """
-        SELECT array_sort(CAST(NULL AS ARRAY), (l, r) -> l - r) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: Constant-zero comparator keeps input order
-      When query
-        """
-        SELECT array_sort(array(3, 1, 2), (l, r) -> 0) AS result
-        """
-      Then query result
-        | result    |
-        | [3, 1, 2] |
-
-  Rule: Comparator form — multi-row
-
-    Scenario: Comparator applied per row
-      When query
-        """
-        SELECT array_sort(a, (l, r) -> l - r) AS result
-        FROM VALUES (array(3, 1, 2)), (array(9, 7, 8)), (CAST(NULL AS ARRAY)) AS t(a)
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-        | [7, 8, 9] |
-        | NULL      |
-
-  Rule: Comparator form — errors
-
-    Scenario: Comparator returning null raises an error
-      When query
-        """
-        SELECT array_sort(array(2, 1), (l, r) -> CAST(NULL AS INT)) AS result
-        """
-      Then query error .*
-
-    Scenario: Comparator returning a non-integer type is rejected
-      When query
-        """
-        SELECT array_sort(array(2, 1), (l, r) -> CAST(l - r AS BIGINT)) AS result
-        """
-      Then query error .*
-
-    Scenario: Comparator with one parameter is rejected
-      When query
-        """
-        SELECT array_sort(array(2, 1), x -> x) AS result
-        """
-      Then query error .*
-
-    Scenario: Comparator with three parameters is rejected
-      When query
-        """
-        SELECT array_sort(array(2, 1), (a, b, c) -> 1) AS result
-        """
-      Then query error .*
-
-    Scenario: Non-array first argument is rejected
-      When query
-        """
-        SELECT array_sort(5, (l, r) -> 1) AS result
-        """
-      Then query error .*
diff --git a/python/pysail/tests/spark/function/features/arrays_zip.feature b/python/pysail/tests/spark/function/features/arrays_zip.feature
deleted file mode 100644
index c1ad65077..000000000
--- a/python/pysail/tests/spark/function/features/arrays_zip.feature
+++ /dev/null
@@ -1,437 +0,0 @@
-Feature: arrays_zip comprehensive tests
-
-  Rule: Basic usage
-
-    Scenario: arrays_zip two arrays same length
-      When query
-        """
-        SELECT arrays_zip(array(1,2,3), array('a','b','c')) AS result
-        """
-      Then query result
-        | result                   |
-        | [{1, a}, {2, b}, {3, c}] |
-
-    Scenario: arrays_zip three arrays
-      When query
-        """
-        SELECT arrays_zip(array(1,2), array('a','b'), array(true,false)) AS result
-        """
-      Then query result
-        | result                        |
-        | [{1, a, true}, {2, b, false}] |
-
-    Scenario: arrays_zip single array
-      When query
-        """
-        SELECT arrays_zip(array(1,2,3)) AS result
-        """
-      Then query result
-        | result          |
-        | [{1}, {2}, {3}] |
-
-    Scenario: arrays_zip four args asymmetric
-      When query
-        """
-        SELECT arrays_zip(array(1), array('a','b'), array(true, false, NULL), array(1.0)) AS result
-        """
-      Then query result
-        | result                                                              |
-        | [{1, a, true, 1.0}, {NULL, b, false, NULL}, {NULL, NULL, NULL, NULL}] |
-
-    Scenario: arrays_zip self-zip same column
-      When query
-        """
-        SELECT arrays_zip(a, a) AS result FROM VALUES (array(1,2,3)) AS t(a)
-        """
-      Then query result
-        | result                   |
-        | [{1, 1}, {2, 2}, {3, 3}] |
-
-    Scenario: arrays_zip zero args returns empty array
-      When query
-        """
-        SELECT arrays_zip() AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-  Rule: Different array lengths
-
-    Scenario: arrays_zip first longer pads NULL
-      When query
-        """
-        SELECT arrays_zip(array(1,2,3), array('a','b')) AS result
-        """
-      Then query result
-        | result                      |
-        | [{1, a}, {2, b}, {3, NULL}] |
-
-    Scenario: arrays_zip second longer pads NULL
-      When query
-        """
-        SELECT arrays_zip(array(1,2), array('a','b','c')) AS result
-        """
-      Then query result
-        | result                      |
-        | [{1, a}, {2, b}, {NULL, c}] |
-
-    Scenario: arrays_zip one empty array
-      When query
-        """
-        SELECT arrays_zip(array(1,2), array()) AS result
-        """
-      Then query result
-        | result                 |
-        | [{1, NULL}, {2, NULL}] |
-
-    Scenario: arrays_zip both empty arrays
-      When query
-        """
-        SELECT arrays_zip(array(), array()) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: arrays_zip four args all empty returns empty
-      When query
-        """
-        SELECT arrays_zip(
-          CAST(array() AS ARRAY),
-          CAST(array() AS ARRAY),
-          CAST(array() AS ARRAY),
-          CAST(array() AS ARRAY)
-        ) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: arrays_zip very asymmetric 1 vs 5
-      When query
-        """
-        SELECT arrays_zip(array(1), array('a','b','c','d','e')) AS result
-        """
-      Then query result
-        | result                                               |
-        | [{1, a}, {NULL, b}, {NULL, c}, {NULL, d}, {NULL, e}] |
-
-  Rule: NULL handling
-
-    Scenario: arrays_zip untyped NULL array returns NULL
-      When query
-        """
-        SELECT arrays_zip(NULL, array(1,2)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: arrays_zip typed NULL array returns NULL
-      When query
-        """
-        SELECT arrays_zip(CAST(NULL AS ARRAY), array(1,2)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: arrays_zip with NULL elements
-      When query
-        """
-        SELECT arrays_zip(array(1,NULL,3), array('a','b','c')) AS result
-        """
-      Then query result
-        | result                      |
-        | [{1, a}, {NULL, b}, {3, c}] |
-
-    Scenario: arrays_zip all NULL elements
-      When query
-        """
-        SELECT arrays_zip(array(NULL,NULL), array(NULL,NULL)) AS result
-        """
-      Then query result
-        | result                       |
-        | [{NULL, NULL}, {NULL, NULL}] |
-
-    Scenario: arrays_zip both args NULL returns NULL
-      When query
-        """
-        SELECT arrays_zip(CAST(NULL AS ARRAY), CAST(NULL AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: arrays_zip four NULL args returns NULL
-      When query
-        """
-        SELECT arrays_zip(
-          CAST(NULL AS ARRAY),
-          CAST(NULL AS ARRAY),
-          CAST(NULL AS ARRAY),
-          CAST(NULL AS ARRAY)
-        ) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: arrays_zip NULL and empty returns NULL
-      When query
-        """
-        SELECT arrays_zip(CAST(NULL AS ARRAY), CAST(array() AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: arrays_zip untyped empty array() pads first as NULL
-      When query
-        """
-        SELECT arrays_zip(array(), array(1,2)) AS result
-        """
-      Then query result
-        | result                 |
-        | [{NULL, 1}, {NULL, 2}] |
-
-  Rule: Type variety
-
-    Scenario: arrays_zip int and double
-      When query
-        """
-        SELECT arrays_zip(array(1,2), array(1.5,2.5)) AS result
-        """
-      Then query result
-        | result                |
-        | [{1, 1.5}, {2, 2.5}]  |
-
-    Scenario: arrays_zip nested arrays
-      When query
-        """
-        SELECT arrays_zip(array(array(1,2),array(3,4)), array('a','b')) AS result
-        """
-      Then query result
-        | result                     |
-        | [{[1, 2], a}, {[3, 4], b}] |
-
-    Scenario: arrays_zip boolean and int
-      When query
-        """
-        SELECT arrays_zip(array(true,false), array(1,2)) AS result
-        """
-      Then query result
-        | result                  |
-        | [{true, 1}, {false, 2}] |
-
-    Scenario: arrays_zip struct elements
-      When query
-        """
-        SELECT arrays_zip(array(struct(1 AS a)), array(struct('x' AS b))) AS result
-        """
-      Then query result
-        | result       |
-        | [{{1}, {x}}] |
-
-    Scenario: arrays_zip map elements
-      When query
-        """
-        SELECT arrays_zip(array(map(1,'a')), array(map(2,'b'))) AS result
-        """
-      Then query result
-        | result                 |
-        | [{{1 -> a}, {2 -> b}}] |
-
-    Scenario: arrays_zip binary elements
-      When query
-        """
-        SELECT arrays_zip(array(X'01', X'02'), array(X'0A', X'0B')) AS result
-        """
-      Then query result
-        | result                       |
-        | [{[01], [0A]}, {[02], [0B]}] |
-
-    Scenario: arrays_zip date elements
-      When query
-        """
-        SELECT arrays_zip(array(DATE'2024-01-01'), array(DATE'2024-12-31')) AS result
-        """
-      Then query result
-        | result                     |
-        | [{2024-01-01, 2024-12-31}] |
-
-    Scenario: arrays_zip timestamp elements
-      When query
-        """
-        SELECT arrays_zip(array(TIMESTAMP'2024-01-01 00:00:00'), array(TIMESTAMP'2024-12-31 23:59:59')) AS result
-        """
-      Then query result
-        | result                                       |
-        | [{2024-01-01 00:00:00, 2024-12-31 23:59:59}] |
-
-    Scenario: arrays_zip decimal elements
-      When query
-        """
-        SELECT arrays_zip(
-          array(CAST(1.5 AS DECIMAL(10,2)), CAST(2.5 AS DECIMAL(10,2))),
-          array(1, 2)
-        ) AS result
-        """
-      Then query result
-        | result                 |
-        | [{1.50, 1}, {2.50, 2}] |
-
-    Scenario: arrays_zip float32 elements
-      When query
-        """
-        SELECT arrays_zip(
-          array(CAST(1.5 AS FLOAT)),
-          array(CAST(2.5 AS FLOAT))
-        ) AS result
-        """
-      Then query result
-        | result       |
-        | [{1.5, 2.5}] |
-
-    Scenario: arrays_zip deeply nested arrays
-      When query
-        """
-        SELECT arrays_zip(array(array(1,2)), array(array(3,4))) AS result
-        """
-      Then query result
-        | result             |
-        | [{[1, 2], [3, 4]}] |
-
-    Scenario: arrays_zip sequence results
-      When query
-        """
-        SELECT arrays_zip(sequence(1, 3), sequence(10, 12)) AS result
-        """
-      Then query result
-        | result                      |
-        | [{1, 10}, {2, 11}, {3, 12}] |
-
-  Rule: Composition
-
-    Scenario: arrays_zip nested in arrays_zip
-      When query
-        """
-        SELECT arrays_zip(arrays_zip(array(1,2), array('a','b')), array(true, false)) AS result
-        """
-      Then query result
-        | result                            |
-        | [{{1, a}, true}, {{2, b}, false}] |
-
-    Scenario: arrays_zip element field access by index-name
-      When query
-        """
-        SELECT arrays_zip(array(1,2,3), array('x','y','z'))[1].`0` AS result
-        """
-      Then query result
-        | result |
-        | 2      |
-
-    Scenario: arrays_zip element field access second position
-      When query
-        """
-        SELECT arrays_zip(array(1,2,3), array('x','y','z'))[1].`1` AS result
-        """
-      Then query result
-        | result |
-        | y      |
-
-    Scenario: arrays_zip to_json roundtrip
-      When query
-        """
-        SELECT to_json(arrays_zip(array(1,2), array('a','b'))) AS result
-        """
-      Then query result
-        | result                            |
-        | [{"0":1,"1":"a"},{"0":2,"1":"b"}] |
-
-    Scenario: arrays_zip LATERAL VIEW explode flatten
-      When query
-        """
-        SELECT e.`0` AS x, e.`1` AS y FROM
-          (SELECT arrays_zip(array(1,2,3), array('a','b','c')) AS z) LATERAL VIEW explode(z) AS e
-        """
-      Then query result ordered
-        | x | y |
-        | 1 | a |
-        | 2 | b |
-        | 3 | c |
-
-  Rule: Multi-row
-
-    Scenario: arrays_zip multi-row
-      When query
-        """
-        SELECT arrays_zip(a, b) AS result FROM VALUES (array(1,2), array('x','y')), (array(3), array('z','w')), (CAST(NULL AS ARRAY), array('a')) AS t(a, b)
-        """
-      Then query result
-        | result              |
-        | [{1, x}, {2, y}]   |
-        | [{3, z}, {NULL, w}] |
-        | NULL                |
-
-    Scenario: arrays_zip multi-row uneven lengths per row
-      When query
-        """
-        SELECT arrays_zip(a, b) AS result FROM VALUES
-          (array(1,2,3), array('x','y')),
-          (array(1), array('a','b','c','d','e')),
-          (array(), array('z'))
-        AS t(a, b)
-        """
-      Then query result
-        | result                                                |
-        | [{1, x}, {2, y}, {3, NULL}]                          |
-        | [{1, a}, {NULL, b}, {NULL, c}, {NULL, d}, {NULL, e}] |
-        | [{NULL, z}]                                           |
-
-    Scenario: arrays_zip multi-row empty row mixed with NULL
-      When query
-        """
-        SELECT arrays_zip(a, b) AS result FROM VALUES
-          (array(), CAST(array() AS ARRAY)),
-          (array(1), array('x')),
-          (CAST(NULL AS ARRAY), CAST(NULL AS ARRAY))
-        AS t(a, b)
-        """
-      Then query result
-        | result   |
-        | []       |
-        | [{1, x}] |
-        | NULL     |
-
-    Scenario: arrays_zip multi-row all-null column returns all NULL
-      When query
-        """
-        SELECT arrays_zip(a, b) AS result FROM VALUES
-          (CAST(NULL AS ARRAY), array(1, 2)),
-          (CAST(NULL AS ARRAY), array(3, 4)),
-          (CAST(NULL AS ARRAY), array(5, 6))
-        AS t(a, b)
-        """
-      Then query result
-        | result |
-        | NULL   |
-        | NULL   |
-        | NULL   |
-
-  Rule: Error conditions
-
-    Scenario: arrays_zip non-array input errors
-      When query
-        """
-        SELECT arrays_zip(1, 2) AS result
-        """
-      Then query error .*
-
-    Scenario: arrays_zip mixed array and non-array errors
-      When query
-        """
-        SELECT arrays_zip(array(1,2), 'hello') AS result
-        """
-      Then query error .*
diff --git a/python/pysail/tests/spark/function/features/base64.feature b/python/pysail/tests/spark/function/features/base64.feature
deleted file mode 100644
index 04f571f69..000000000
--- a/python/pysail/tests/spark/function/features/base64.feature
+++ /dev/null
@@ -1,71 +0,0 @@
-Feature: base64 functions encode and decode binary strings
-
-  Rule: Null propagation
-
-    Scenario: base64 returns null for a null string
-      When query
-      """
-      SELECT base64(CAST(NULL AS STRING)) AS result
-      """
-      Then query result
-      | result |
-      | NULL   |
-
-    Scenario: base64 returns null for an untyped null literal
-      When query
-      """
-      SELECT base64(NULL) AS result
-      """
-      Then query result
-      | result |
-      | NULL   |
-
-    Scenario: base64 preserves nulls in column values
-      When query
-      """
-      SELECT base64(value) AS result
-      FROM VALUES ('ab'), (CAST(NULL AS STRING)) AS data(value)
-      ORDER BY value IS NULL, value
-      """
-      Then query result
-      | result |
-      | YWI=   |
-      | NULL   |
-
-  Rule: Null-tolerant decoding
-
-    Scenario: unbase64 returns null for a null string
-      When query
-      """
-      SELECT unbase64(CAST(NULL AS STRING)) AS result
-      """
-      Then query result
-      | result |
-      | NULL   |
-
-    Scenario: unbase64 returns null for an untyped null literal
-      When query
-      """
-      SELECT unbase64(NULL) AS result
-      """
-      Then query result
-      | result |
-      | NULL   |
-
-    Scenario: unbase64 ignores whitespace and decodes unpadded input
-      When query
-      """
-      SELECT unbase64('   ab   ') AS result
-      """
-      Then query result
-      | result |
-      | [69]   |
-
-    Scenario: unbase64 ignores non-base64 characters
-      When query
-      """
-      SELECT unbase64('%') AS result
-      """
-      Then query result
-      | result |
-      | []     |
diff --git a/python/pysail/tests/spark/function/features/bin.feature b/python/pysail/tests/spark/function/features/bin.feature
deleted file mode 100644
index 53de7d3ba..000000000
--- a/python/pysail/tests/spark/function/features/bin.feature
+++ /dev/null
@@ -1,357 +0,0 @@
-Feature: bin converts integral values to binary strings
-
-  Rule: String arguments follow Spark cast semantics
-
-    Scenario: bin trims strings before casting to integers
-      When query
-        """
-        SELECT bin(value) AS result
-        FROM VALUES (' 13 '), (' -13 ') AS data(value)
-        ORDER BY value
-        """
-      Then query result
-        | result                                                           |
-        | 1111111111111111111111111111111111111111111111111111111111110011 |
-        | 1101                                                             |
-
-    Scenario: bin malformed string errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin('ab') AS result
-        """
-      Then query error CAST_INVALID_INPUT
-
-    Scenario: bin malformed string returns NULL under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin(value) AS result
-        FROM VALUES ('ab'), (CAST(NULL AS STRING)) AS data(value)
-        ORDER BY value IS NULL, value
-        """
-      Then query result
-        | result |
-        | NULL   |
-        | NULL   |
-
-    Scenario: bin empty string errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin('') AS result
-        """
-      Then query error CAST_INVALID_INPUT
-
-    Scenario: bin empty string returns NULL under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin('') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: bin decimal string errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin('13.9') AS result
-        """
-      Then query error CAST_INVALID_INPUT
-
-    Scenario: bin decimal string truncates toward zero under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin(value) AS result
-        FROM VALUES (0, '13.9'), (1, '-13.9'), (2, '.3') AS data(id, value)
-        ORDER BY id
-        """
-      Then query result
-        | result                                                           |
-        | 1101                                                             |
-        | 1111111111111111111111111111111111111111111111111111111111110011 |
-        | 0                                                                |
-
-    Scenario: bin out-of-range string errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin('99999999999999999999') AS result
-        """
-      Then query error CAST_INVALID_INPUT
-
-    Scenario: bin out-of-range string returns NULL under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin('99999999999999999999') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: bin scientific notation string errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin('1e3') AS result
-        """
-      Then query error CAST_INVALID_INPUT
-
-    Scenario: bin scientific notation strings return NULL under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin(value) AS result FROM VALUES
-          (0, '1e3'),
-          (1, '1E3'),
-          (2, '  1e3  ')
-        AS t(id, value) ORDER BY id
-        """
-      Then query result
-        | result |
-        | NULL   |
-        | NULL   |
-        | NULL   |
-
-  Rule: Integer boundaries
-
-    Scenario: bin zero
-      When query
-        """
-        SELECT bin(0) AS result
-        """
-      Then query result
-        | result |
-        | 0      |
-
-    Scenario: bin minus one is all ones
-      When query
-        """
-        SELECT bin(-1) AS result
-        """
-      Then query result
-        | result                                                           |
-        | 1111111111111111111111111111111111111111111111111111111111111111 |
-
-    Scenario: bin INT_MAX uses 31 bits
-      When query
-        """
-        SELECT bin(2147483647) AS result
-        """
-      Then query result
-        | result                          |
-        | 1111111111111111111111111111111 |
-
-    Scenario: bin INT_MIN sign-extends to 64 bits
-      When query
-        """
-        SELECT bin(-2147483648) AS result
-        """
-      Then query result
-        | result                                                           |
-        | 1111111111111111111111111111111110000000000000000000000000000000 |
-
-    Scenario: bin LONG_MAX uses 63 bits
-      When query
-        """
-        SELECT bin(9223372036854775807L) AS result
-        """
-      Then query result
-        | result                                                          |
-        | 111111111111111111111111111111111111111111111111111111111111111 |
-
-    Scenario: bin LONG_MIN is 1 followed by 63 zeros
-      When query
-        """
-        SELECT bin(CAST(-9223372036854775808 AS BIGINT)) AS result
-        """
-      Then query result
-        | result                                                           |
-        | 1000000000000000000000000000000000000000000000000000000000000000 |
-
-    Scenario: bin TINYINT input promotes to BIGINT semantics
-      When query
-        """
-        SELECT bin(CAST(99 AS TINYINT)) AS result
-        """
-      Then query result
-        | result  |
-        | 1100011 |
-
-  Rule: Floating-point truncation toward zero
-
-    Scenario: bin DOUBLE 1.5 truncates to 1
-      When query
-        """
-        SELECT bin(1.5) AS result
-        """
-      Then query result
-        | result |
-        | 1      |
-
-    Scenario: bin DOUBLE 0.5 truncates to 0
-      When query
-        """
-        SELECT bin(0.5) AS result
-        """
-      Then query result
-        | result |
-        | 0      |
-
-    Scenario: bin DOUBLE -0.5 truncates toward zero
-      When query
-        """
-        SELECT bin(-0.5) AS result
-        """
-      Then query result
-        | result |
-        | 0      |
-
-    Scenario: bin DOUBLE -1.5 truncates toward zero then sign-extends
-      When query
-        """
-        SELECT bin(-1.5) AS result
-        """
-      Then query result
-        | result                                                           |
-        | 1111111111111111111111111111111111111111111111111111111111111111 |
-
-    Scenario: bin DECIMAL truncates fractional part
-      When query
-        """
-        SELECT bin(CAST(1.5 AS DECIMAL(3,2))) AS result
-        """
-      Then query result
-        | result |
-        | 1      |
-
-  Rule: NULL inputs
-
-    Scenario: bin untyped NULL returns NULL
-      When query
-        """
-        SELECT bin(NULL) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: bin typed NULL BIGINT returns NULL
-      When query
-        """
-        SELECT bin(CAST(NULL AS BIGINT)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Unsupported types are rejected
-
-    Scenario: bin rejects BOOLEAN
-      When query
-        """
-        SELECT bin(true) AS result
-        """
-      Then query error .*
-
-    Scenario: bin rejects DATE
-      When query
-        """
-        SELECT bin(DATE '2024-01-15') AS result
-        """
-      Then query error .*
-
-    Scenario: bin rejects TIMESTAMP
-      When query
-        """
-        SELECT bin(TIMESTAMP '2024-01-15 12:00:00') AS result
-        """
-      Then query error .*
-
-    Scenario: bin rejects BINARY
-      When query
-        """
-        SELECT bin(X'01') AS result
-        """
-      Then query error .*
-
-    Scenario: bin rejects ARRAY
-      When query
-        """
-        SELECT bin(array(1)) AS result
-        """
-      Then query error .*
-
-    Scenario: bin rejects MAP
-      When query
-        """
-        SELECT bin(map('a', 1)) AS result
-        """
-      Then query error .*
-
-    Scenario: bin rejects STRUCT
-      When query
-        """
-        SELECT bin(named_struct('a', 1)) AS result
-        """
-      Then query error .*
-
-  Rule: Float NaN and Infinity follow cast semantics
-
-    Scenario: bin NaN errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin(CAST('NaN' AS DOUBLE)) AS result
-        """
-      Then query error .*
-
-    Scenario: bin NaN truncates to zero under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin(CAST('NaN' AS DOUBLE)) AS result
-        """
-      Then query result
-        | result |
-        | 0      |
-
-    Scenario: bin Infinity errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin(CAST('Infinity' AS DOUBLE)) AS result
-        """
-      Then query error .*
-
-    Scenario: bin Infinity saturates to LONG_MAX under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin(CAST('Infinity' AS DOUBLE)) AS result
-        """
-      Then query result
-        | result                                                          |
-        | 111111111111111111111111111111111111111111111111111111111111111 |
-
-    Scenario: bin out-of-range DOUBLE errors under ANSI on
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT bin(CAST(1e30 AS DOUBLE)) AS result
-        """
-      Then query error CAST_OVERFLOW
-
-    Scenario: bin out-of-range DOUBLE saturates to LONG_MAX under ANSI off
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT bin(CAST(1e30 AS DOUBLE)) AS result
-        """
-      Then query result
-        | result                                                          |
-        | 111111111111111111111111111111111111111111111111111111111111111 |
diff --git a/python/pysail/tests/spark/function/features/bitwise/bit_count.feature b/python/pysail/tests/spark/function/features/bitwise/bit_count.feature
new file mode 100644
index 000000000..2db39944c
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/bitwise/bit_count.feature
@@ -0,0 +1,37 @@
+Feature: bit_count output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to bit_count yields the schema Spark declares
+      When query
+        """
+        SELECT bit_count(0) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a non-null column input to bit_count yields the schema Spark declares
+      When query
+        """
+        SELECT bit_count(CAST(id AS INT)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to bit_count stays nullable
+      When query
+        """
+        SELECT bit_count(c) AS result FROM VALUES (0), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/bitwise/bit_get.feature b/python/pysail/tests/spark/function/features/bitwise/bit_get.feature
new file mode 100644
index 000000000..09f94d605
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/bitwise/bit_get.feature
@@ -0,0 +1,37 @@
+Feature: bit_get output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to bit_get yields the schema Spark declares
+      When query
+        """
+        SELECT bit_get(11, 0) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: byte (nullable = false)
+        """
+
+    Scenario: a non-null column input to bit_get yields the schema Spark declares
+      When query
+        """
+        SELECT bit_get(CAST(id AS INT), 0) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: byte (nullable = false)
+        """
+
+    Scenario: a nullable column input to bit_get stays nullable
+      When query
+        """
+        SELECT bit_get(c, 0) AS result FROM VALUES (11), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: byte (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/bitwise/getbit.feature b/python/pysail/tests/spark/function/features/bitwise/getbit.feature
new file mode 100644
index 000000000..9c78cb6ac
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/bitwise/getbit.feature
@@ -0,0 +1,37 @@
+Feature: getbit output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to getbit yields the schema Spark declares
+      When query
+        """
+        SELECT getbit(11, 0) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: byte (nullable = false)
+        """
+
+    Scenario: a non-null column input to getbit yields the schema Spark declares
+      When query
+        """
+        SELECT getbit(CAST(id AS INT), 0) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: byte (nullable = false)
+        """
+
+    Scenario: a nullable column input to getbit stays nullable
+      When query
+        """
+        SELECT getbit(c, 0) AS result FROM VALUES (11), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: byte (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/bitwise/shiftleft.feature b/python/pysail/tests/spark/function/features/bitwise/shiftleft.feature
new file mode 100644
index 000000000..83cb33288
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/bitwise/shiftleft.feature
@@ -0,0 +1,37 @@
+Feature: shiftleft output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to shiftleft yields the schema Spark declares
+      When query
+        """
+        SELECT shiftleft(2, 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a non-null column input to shiftleft yields the schema Spark declares
+      When query
+        """
+        SELECT shiftleft(CAST(id AS INT), 1) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to shiftleft stays nullable
+      When query
+        """
+        SELECT shiftleft(c, 1) AS result FROM VALUES (2), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/bitwise/shiftright.feature b/python/pysail/tests/spark/function/features/bitwise/shiftright.feature
new file mode 100644
index 000000000..b589338ae
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/bitwise/shiftright.feature
@@ -0,0 +1,37 @@
+Feature: shiftright output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to shiftright yields the schema Spark declares
+      When query
+        """
+        SELECT shiftright(4, 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a non-null column input to shiftright yields the schema Spark declares
+      When query
+        """
+        SELECT shiftright(CAST(id AS INT), 1) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to shiftright stays nullable
+      When query
+        """
+        SELECT shiftright(c, 1) AS result FROM VALUES (4), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/bitwise/shiftrightunsigned.feature b/python/pysail/tests/spark/function/features/bitwise/shiftrightunsigned.feature
new file mode 100644
index 000000000..557fec5ae
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/bitwise/shiftrightunsigned.feature
@@ -0,0 +1,39 @@
+Feature: shiftrightunsigned output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to shiftrightunsigned yields the schema Spark declares
+      When query
+        """
+        SELECT shiftrightunsigned(4, 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null column input to shiftrightunsigned yields the schema Spark declares
+      When query
+        """
+        SELECT shiftrightunsigned(CAST(id AS INT), 1) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to shiftrightunsigned stays nullable
+      When query
+        """
+        SELECT shiftrightunsigned(c, 1) AS result FROM VALUES (4), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/cast_date_to_numeric.feature b/python/pysail/tests/spark/function/features/cast_date_to_numeric.feature
deleted file mode 100644
index a112d9c0a..000000000
--- a/python/pysail/tests/spark/function/features/cast_date_to_numeric.feature
+++ /dev/null
@@ -1,128 +0,0 @@
-Feature: CAST date to numeric types returns null
-
-  In Spark legacy mode, casting a DATE to any numeric type returns NULL.
-  In ANSI mode, the cast raises an error. TRY_CAST always returns NULL.
-
-  Rule: CAST date to numeric types returns null (legacy mode)
-
-    Scenario: cast date to int returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS INT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to bigint returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS BIGINT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to smallint returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS SMALLINT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to tinyint returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS TINYINT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to float returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS FLOAT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to double returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS DOUBLE) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to decimal returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS DECIMAL(10,2)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast date to boolean returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS BOOLEAN) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: cast null date to int returns null
-      Given config spark.sql.ansi.enabled = false
-      When query
-        """
-        SELECT CAST(CAST(NULL AS DATE) AS INT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: CAST date to numeric in ANSI mode raises error
-
-    @sail-only
-    Scenario: cast date to int in ANSI mode raises error
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS INT) AS result
-        """
-      Then query error cannot cast date
-      Given config spark.sql.ansi.enabled = false
-
-    @sail-only
-    Scenario: cast date to double in ANSI mode raises error
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS DOUBLE) AS result
-        """
-      Then query error cannot cast date
-      Given config spark.sql.ansi.enabled = false
-
-    @sail-only
-    Scenario: cast date to boolean in ANSI mode raises error
-      Given config spark.sql.ansi.enabled = true
-      When query
-        """
-        SELECT CAST(DATE '2023-01-15' AS BOOLEAN) AS result
-        """
-      Then query error cannot cast date
-      Given config spark.sql.ansi.enabled = false
diff --git a/python/pysail/tests/spark/function/features/cast_nan_inf.feature b/python/pysail/tests/spark/function/features/cast_nan_inf.feature
deleted file mode 100644
index bad64efa5..000000000
--- a/python/pysail/tests/spark/function/features/cast_nan_inf.feature
+++ /dev/null
@@ -1,291 +0,0 @@
-Feature: CAST and type constructors with NaN and Infinity (issue #630)
-
-  Rule: FLOAT type constructor
-
-    Scenario: FLOAT NaN
-      When query
-        """
-        SELECT FLOAT('NAN') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: FLOAT NaN lowercase
-      When query
-        """
-        SELECT FLOAT('nan') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: FLOAT NaN mixed case
-      When query
-        """
-        SELECT FLOAT('Nan') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    @sail-bug
-    # Sail does not trim spaces before parsing NaN
-    Scenario: FLOAT NaN with spaces
-      When query
-        """
-        SELECT FLOAT(' NaN ') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: FLOAT negative NaN
-      When query
-        """
-        SELECT FLOAT('-NaN') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: FLOAT Infinity
-      When query
-        """
-        SELECT FLOAT('Infinity') AS result
-        """
-      Then query result
-        | result   |
-        | Infinity |
-
-    Scenario: FLOAT negative Infinity
-      When query
-        """
-        SELECT FLOAT('-Infinity') AS result
-        """
-      Then query result
-        | result    |
-        | -Infinity |
-
-    Scenario: FLOAT Infinity lowercase
-      When query
-        """
-        SELECT FLOAT('infinity') AS result
-        """
-      Then query result
-        | result   |
-        | Infinity |
-
-    Scenario: FLOAT normal value
-      When query
-        """
-        SELECT FLOAT('42') AS result
-        """
-      Then query result
-        | result |
-        | 42.0   |
-
-  Rule: DOUBLE type constructor
-
-    Scenario: DOUBLE NaN
-      When query
-        """
-        SELECT DOUBLE('NAN') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: DOUBLE Infinity uppercase
-      When query
-        """
-        SELECT DOUBLE('INFINITY') AS result
-        """
-      Then query result
-        | result   |
-        | Infinity |
-
-    Scenario: DOUBLE negative Infinity uppercase
-      When query
-        """
-        SELECT DOUBLE('-INFINITY') AS result
-        """
-      Then query result
-        | result    |
-        | -Infinity |
-
-    @sail-bug
-    # Sail does not trim spaces before parsing Infinity
-    Scenario: DOUBLE Infinity with spaces
-      When query
-        """
-        SELECT DOUBLE(' Infinity ') AS result
-        """
-      Then query result
-        | result   |
-        | Infinity |
-
-    Scenario: DOUBLE normal value
-      When query
-        """
-        SELECT DOUBLE('3.14') AS result
-        """
-      Then query result
-        | result |
-        | 3.14   |
-
-  Rule: CAST to FLOAT/DOUBLE
-
-    Scenario: CAST NaN to FLOAT
-      When query
-        """
-        SELECT CAST('NaN' AS FLOAT) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: CAST NaN to DOUBLE
-      When query
-        """
-        SELECT CAST('NaN' AS DOUBLE) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-  Rule: Integer types reject NaN and Infinity
-
-    Scenario: INT NaN errors
-      When query
-        """
-        SELECT INT('NAN') AS result
-        """
-      Then query error .*
-
-    Scenario: CAST NaN to INT errors
-      When query
-        """
-        SELECT CAST('NaN' AS INT) AS result
-        """
-      Then query error .*
-
-    Scenario: INT Infinity errors
-      When query
-        """
-        SELECT INT('Infinity') AS result
-        """
-      Then query error .*
-
-    Scenario: BIGINT NaN errors
-      When query
-        """
-        SELECT BIGINT('NaN') AS result
-        """
-      Then query error .*
-
-    Scenario: SMALLINT NaN errors
-      When query
-        """
-        SELECT SMALLINT('NaN') AS result
-        """
-      Then query error .*
-
-    Scenario: TINYINT NaN errors
-      When query
-        """
-        SELECT TINYINT('NaN') AS result
-        """
-      Then query error .*
-
-    Scenario: DECIMAL NaN errors
-      When query
-        """
-        SELECT CAST('NaN' AS DECIMAL(10,2)) AS result
-        """
-      Then query error .*
-
-    Scenario: INT invalid string errors
-      When query
-        """
-        SELECT INT('hello') AS result
-        """
-      Then query error .*
-
-  Rule: TRY_CAST with NaN
-
-    Scenario: TRY_CAST NaN to INT returns NULL
-      When query
-        """
-        SELECT TRY_CAST('NaN' AS INT) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: TRY_CAST NaN to FLOAT returns NaN
-      When query
-        """
-        SELECT TRY_CAST('NaN' AS FLOAT) AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-  Rule: NaN arithmetic
-
-    Scenario: NaN plus number is NaN
-      When query
-        """
-        SELECT FLOAT('NaN') + 1 AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: NaN equals NaN is true in Spark
-      When query
-        """
-        SELECT FLOAT('NaN') = FLOAT('NaN') AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: NaN greater than zero is true in Spark
-      When query
-        """
-        SELECT FLOAT('NaN') > 0 AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-  Rule: Multi-row with NaN and Infinity
-
-    Scenario: VALUES with FLOAT NaN Infinity and NULL
-      When query
-        """
-        SELECT * FROM VALUES (FLOAT('NaN')), (FLOAT('Infinity')), (FLOAT('-Infinity')), (NULL), (0.0), (1.5) AS t(v)
-        """
-      Then query result
-        | v         |
-        | NaN       |
-        | Infinity  |
-        | -Infinity |
-        | NULL      |
-        | 0.0       |
-        | 1.5       |
-
-    Scenario: VALUES with DOUBLE NaN Infinity and NULL
-      When query
-        """
-        SELECT * FROM VALUES (DOUBLE('NaN')), (DOUBLE('Infinity')), (DOUBLE('-Infinity')), (NULL), (0.0) AS t(v)
-        """
-      Then query result
-        | v         |
-        | NaN       |
-        | Infinity  |
-        | -Infinity |
-        | NULL      |
-        | 0.0       |
diff --git a/python/pysail/tests/spark/function/features/coalesce.feature b/python/pysail/tests/spark/function/features/coalesce.feature
deleted file mode 100644
index 5c4851e0f..000000000
--- a/python/pysail/tests/spark/function/features/coalesce.feature
+++ /dev/null
@@ -1,115 +0,0 @@
-Feature: coalesce returns the first non-null argument
-
-  Rule: Spark-compatible coercion for mixed string and temporal arguments
-
-    Scenario: Coalesce null string column falls back to date column as string
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      WITH t(string_col, date_col) AS (
-        SELECT CAST(NULL AS STRING), DATE '2024-01-15'
-      )
-      SELECT
-        coalesce(string_col, date_col) AS result,
-        typeof(coalesce(string_col, date_col)) AS result_type
-      FROM t
-      """
-      Then query result
-      | result     | result_type |
-      | 2024-01-15 | string      |
-
-    Scenario: Coalesce null string column falls back to timestamp column as string
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      WITH t(string_col, timestamp_col) AS (
-        SELECT CAST(NULL AS STRING), TIMESTAMP '2024-01-15 10:30:00'
-      )
-      SELECT
-        coalesce(string_col, timestamp_col) AS result,
-        typeof(coalesce(string_col, timestamp_col)) AS result_type
-      FROM t
-      """
-      Then query result
-      | result              | result_type |
-      | 2024-01-15 10:30:00 | string      |
-
-    Scenario: Coalesce string literal before a date column wins without temporal casting
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      WITH t(date_col) AS (
-        SELECT DATE '2024-01-15'
-      )
-      SELECT
-        coalesce('default', date_col) AS result,
-        typeof(coalesce('default', date_col)) AS result_type
-      FROM t
-      """
-      Then query result
-      | result  | result_type |
-      | default | string      |
-
-    Scenario: Coalesce non-null string column before a date column wins without temporal casting
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      WITH t(string_col, date_col) AS (
-        SELECT CAST('hello' AS STRING), DATE '2024-01-15'
-      )
-      SELECT
-        coalesce(string_col, date_col) AS result,
-        typeof(coalesce(string_col, date_col)) AS result_type
-      FROM t
-      """
-      Then query result
-      | result | result_type |
-      | hello  | string      |
-
-    Scenario: Coalesce date column before a string column returns a string value
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      WITH t(date_col, string_col) AS (
-        SELECT DATE '2024-01-15', CAST('fallback' AS STRING)
-      )
-      SELECT
-        coalesce(date_col, string_col) AS result,
-        typeof(coalesce(date_col, string_col)) AS result_type
-      FROM t
-      """
-      Then query result
-      | result     | result_type |
-      | 2024-01-15 | string      |
-
-    Scenario: Coalesce multiple mixed temporal arguments still coerces to string
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      SELECT
-        coalesce(
-          CAST(NULL AS STRING),
-          CAST(NULL AS DATE),
-          TIMESTAMP '2024-01-15 10:30:00'
-        ) AS result,
-        typeof(coalesce(
-          CAST(NULL AS STRING),
-          CAST(NULL AS DATE),
-          TIMESTAMP '2024-01-15 10:30:00'
-        )) AS result_type
-      """
-      Then query result
-      | result              | result_type |
-      | 2024-01-15 10:30:00 | string      |
-
-    Scenario: Coalesce all-null mixed string and date arguments returns null with string type
-      Given config spark.sql.ansi.enabled = false
-      When query
-      """
-      SELECT
-        coalesce(CAST(NULL AS STRING), CAST(NULL AS DATE)) AS result,
-        typeof(coalesce(CAST(NULL AS STRING), CAST(NULL AS DATE))) AS result_type
-      """
-      Then query result
-      | result | result_type |
-      | NULL   | string      |
diff --git a/python/pysail/tests/spark/function/features/collection/cardinality.feature b/python/pysail/tests/spark/function/features/collection/cardinality.feature
new file mode 100644
index 000000000..774c4f0e0
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/collection/cardinality.feature
@@ -0,0 +1,27 @@
+Feature: cardinality output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to cardinality yields the schema Spark declares
+      When query
+        """
+        SELECT cardinality(array('b', 'd', 'c', 'a')) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to cardinality stays nullable
+      When query
+        """
+        SELECT cardinality(c) AS result FROM VALUES (array('b', 'd', 'c', 'a')), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/collection/concat.feature b/python/pysail/tests/spark/function/features/collection/concat.feature
new file mode 100644
index 000000000..41dd93426
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/collection/concat.feature
@@ -0,0 +1,336 @@
+Feature: concat function
+
+  Rule: Basic concatenation
+
+    Scenario Outline: Basic: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                      | args                        | result          |
+        | concat two integer arrays | array(1, 2, 3), array(4, 5) | [1, 2, 3, 4, 5] |
+        | concat two string arrays  | array('a', 'b'), array('c') | [a, b, c]       |
+
+  Rule: Empty array handling
+
+    Scenario Outline: Empty array: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                | args                    | result    |
+        | concat empty array with typed array | array(), array(1, 2, 3) | [1, 2, 3] |
+        | concat typed array with empty array | array(1, 2), array()    | [1, 2]    |
+
+  Rule: Null propagation
+
+    Scenario Outline: Array null propagation: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case                                | args                                  |
+        | concat array with null returns null | array(1, 2), CAST(NULL AS ARRAY) |
+        | concat null with array returns null | CAST(NULL AS ARRAY), array(1, 2) |
+
+  Rule: String concatenation
+
+    Scenario Outline: String: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                               | args                                   | result       |
+        | basic string concatenation         | 'Spark', 'SQL'                         | SparkSQL     |
+        | three string arguments             | 'Hello', ', ', 'World'                 | Hello, World |
+        | single string argument             | 'hello'                                | hello        |
+        | empty string with non-empty string | '', 'hello'                            | hello        |
+        | many string arguments              | 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h' | abcdefgh     |
+
+    Scenario Outline: String with empty result: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result |
+        |        |
+
+      Examples:
+        | case                                | args       |
+        | zero arguments returns empty string |            |
+        | empty strings                       | '', ''     |
+        | whitespace strings                  | '  ', '  ' |
+
+  Rule: String NULL propagation
+
+    Scenario Outline: String NULL propagation: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case                                | args                      |
+        | NULL only returns NULL              | NULL                      |
+        | NULL with string returns NULL       | NULL, 'b'                 |
+        | string with NULL returns NULL       | 'a', NULL                 |
+        | NULL and NULL returns NULL          | NULL, NULL                |
+        | string NULL string returns NULL     | 'a', NULL, 'b'            |
+        | typed NULL string returns NULL      | CAST(NULL AS STRING)      |
+        | typed NULL with string returns NULL | CAST(NULL AS STRING), 'b' |
+
+  Rule: Type coercion to string
+
+    Scenario Outline: Type coercion: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                | args                            | result              |
+        | TINYINT coerced to string           | CAST(1 AS TINYINT)              | 1                   |
+        | INT coerced to string               | 1                               | 1                   |
+        | INT and INT concatenated as strings | 1, 2                            | 12                  |
+        | DOUBLE coerced to string            | 1.0                             | 1.0                 |
+        | DECIMAL coerced to string           | CAST(1.0 AS DECIMAL(10,2))      | 1.00                |
+        | BOOLEAN coerced to string           | true                            | true                |
+        | DATE coerced to string              | DATE '2024-01-15'               | 2024-01-15          |
+        | TIMESTAMP coerced to string         | TIMESTAMP '2024-01-15 12:00:00' | 2024-01-15 12:00:00 |
+        | string with INT coercion            | 'hello', 1                      | hello1              |
+
+  Rule: Binary concatenation
+
+    Scenario Outline: Binary: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                        | args                    | result           |
+        | basic binary concatenation  | X'4865', X'6C6C6F'      | [48 65 6C 6C 6F] |
+        | single binary argument      | X'48656C6C6F'           | [48 65 6C 6C 6F] |
+        | three binary arguments      | X'48', X'65', X'6C6C6F' | [48 65 6C 6C 6F] |
+        | empty binary with binary    | X'', X'4865'            | [48 65]          |
+        | binary with string coercion | X'48656C6C6F', 'world'  | Helloworld       |
+        | string with binary coercion | 'hello', X'48656C6C6F'  | helloHello       |
+
+  Rule: Binary NULL propagation
+
+    Scenario Outline: Binary NULL propagation: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case                                 | args                                       |
+        | NULL typed binary returns NULL       | CAST(NULL AS BINARY)                       |
+        | NULL binary with binary returns NULL | CAST(NULL AS BINARY), X'48'                |
+        | binary with NULL binary returns NULL | X'48', CAST(NULL AS BINARY)                |
+        | two NULL binaries returns NULL       | CAST(NULL AS BINARY), CAST(NULL AS BINARY) |
+
+  Rule: Array concatenation
+
+    Scenario Outline: Array: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                | args                                               | result             |
+        | concat two boolean arrays                           | array(true), array(false)                          | [true, false]      |
+        | concat nested arrays                                | array(array(1, 2)), array(array(3, 4))             | [[1, 2], [3, 4]]   |
+        | concat array with null elements                     | array(1, NULL, 3), array(4)                        | [1, NULL, 3, 4]    |
+        | concat three arrays                                 | array(1, 2, 3), array(4, 5), array(6)              | [1, 2, 3, 4, 5, 6] |
+        | concat two empty arrays                             | array(), array()                                   | []                 |
+        | NULL typed array returns NULL                       | CAST(NULL AS ARRAY)                           | NULL               |
+        | typed NULL array with typed NULL array returns NULL | CAST(NULL AS ARRAY), CAST(NULL AS ARRAY) | NULL               |
+
+  Rule: Multi-row behavior
+
+    Scenario Outline: Multi-row: 
+      When query
+        """
+        SELECT concat(a, b) AS result
+        FROM VALUES  AS t(a, b)
+        """
+      Then query result
+        | result  |
+        |  |
+        | NULL    |
+        | NULL    |
+
+      Examples:
+        | case                                | values                                                      | first       |
+        | string concat from table with NULLs | ('hello', ' world'), (NULL, 'x'), ('a', NULL)               | hello world |
+        | array concat from table with NULLs  | (array(1, 2), array(3)), (NULL, array(4)), (array(5), NULL) | [1, 2, 3]   |
+
+  Rule: Error cases
+
+    Scenario Outline: Error: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query error .*
+
+      Examples:
+        | case                                            | args                 |
+        | MAP type is rejected                            | map('a', 1)          |
+        | STRUCT type is rejected                         | named_struct('a', 1) |
+        | untyped NULL mixed with typed array is rejected | NULL, array(1, 2)    |
+        | typed array mixed with untyped NULL is rejected | array(1, 2), NULL    |
+
+  Rule: Timestamp coercion to string
+
+    Scenario Outline: Timestamp: 
+      When query
+        """
+        SELECT concat() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                              | args                                        | result                     |
+        | TIMESTAMP coerced to string with microseconds     | TIMESTAMP '2024-01-15 12:00:00.123456'      | 2024-01-15 12:00:00.123456 |
+        | TIMESTAMP concatenated with string suffix         | TIMESTAMP '2024-01-15 12:00:00', '_suffix'  | 2024-01-15 12:00:00_suffix |
+        | string prefix concatenated with TIMESTAMP         | 'prefix_', TIMESTAMP '2024-01-15 12:00:00'  | prefix_2024-01-15 12:00:00 |
+        | TIMESTAMP NULL with string returns NULL           | CAST(NULL AS TIMESTAMP), 'x'                | NULL                       |
+        | TIMESTAMP_NTZ coerced to string                   | TIMESTAMP_NTZ '2024-01-15 12:00:00'         | 2024-01-15 12:00:00        |
+        | TIMESTAMP_NTZ coerced to string with microseconds | TIMESTAMP_NTZ '2024-01-15 12:00:00.123456'  | 2024-01-15 12:00:00.123456 |
+        | TIMESTAMP_NTZ concatenated with string suffix     | TIMESTAMP_NTZ '2024-01-15 12:00:00', '_end' | 2024-01-15 12:00:00_end    |
+        | TIMESTAMP_NTZ NULL with string returns NULL       | CAST(NULL AS TIMESTAMP_NTZ), 'x'            | NULL                       |
+
+    Scenario: two TIMESTAMPs cast to string concatenated with separator
+      When query
+        """
+        SELECT concat(
+          CAST(TIMESTAMP '2024-01-15 12:00:00' AS STRING),
+          '_',
+          CAST(TIMESTAMP '2024-01-16 13:14:15' AS STRING)
+        ) AS result
+        """
+      Then query result
+        | result                                  |
+        | 2024-01-15 12:00:00_2024-01-16 13:14:15 |
+
+  Rule: Nested concat
+
+    Scenario: nested concat with string constants folds to single value
+      When query
+        """
+        SELECT concat(concat('a', 'b'), 'c') AS result
+        """
+      Then query result
+        | result |
+        | abc    |
+
+    Scenario: nested concat with column flattens and returns correct result
+      When query
+        """
+        SELECT concat(concat(v, 'b'), 'c') AS result
+        FROM VALUES ('a') AS t(v)
+        """
+      Then query result
+        | result |
+        | abc    |
+
+  Rule: Single-argument simplify edge cases
+
+    Scenario Outline: Single argument: 
+      When query
+        """
+        SELECT concat() AS result
+        FROM VALUES () AS t()
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                      | expr               | value                                  | col | result                     |
+        | single timestamp column coerced to string                 | ts                 | TIMESTAMP '2024-01-15 12:00:00'        | ts  | 2024-01-15 12:00:00        |
+        | single timestamp column with microseconds                 | ts                 | TIMESTAMP '2024-01-15 12:00:00.123456' | ts  | 2024-01-15 12:00:00.123456 |
+        | single timestamp column explicit cast matches bare column | CAST(ts AS STRING) | TIMESTAMP '2024-01-15 12:00:00'        | ts  | 2024-01-15 12:00:00        |
+        | single INT column coerced to string                       | n                  | 42                                     | n   | 42                         |
+        | single DOUBLE column coerced to string                    | d                  | 3.14                                   | d   | 3.14                       |
+        | single BOOLEAN column coerced to string                   | b                  | true                                   | b   | true                       |
+        | single NULL timestamp column returns NULL                 | ts                 | CAST(NULL AS TIMESTAMP)                | ts  | NULL                       |
+        | single TIMESTAMP_NTZ column coerced to string             | ts                 | TIMESTAMP_NTZ '2024-01-15 12:00:00'    | ts  | 2024-01-15 12:00:00        |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: non-null string literals yield a non-nullable string
+      When query
+        """
+        SELECT concat('a', 'b') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
+
+    Scenario: a non-null string column yields a non-nullable string
+      When query
+        """
+        SELECT concat(CAST(id AS STRING), 'x') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
+
+    Scenario: a nullable string column stays nullable
+      When query
+        """
+        SELECT concat(c, 'x') AS result FROM VALUES ('a'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/collection/element_at.feature b/python/pysail/tests/spark/function/features/collection/element_at.feature
new file mode 100644
index 000000000..6ec52fede
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/collection/element_at.feature
@@ -0,0 +1,27 @@
+Feature: element_at output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to element_at yields the schema Spark declares
+      When query
+        """
+        SELECT element_at(array(1, 2, 3), 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to element_at stays nullable
+      When query
+        """
+        SELECT element_at(c, 2) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/collection/reverse.feature b/python/pysail/tests/spark/function/features/collection/reverse.feature
new file mode 100644
index 000000000..d02a9c6f0
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/collection/reverse.feature
@@ -0,0 +1,39 @@
+Feature: reverse output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to reverse yields the schema Spark declares
+      When query
+        """
+        SELECT reverse('Spark SQL') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
+
+    @sail-bug
+    Scenario: a non-null column input to reverse yields the schema Spark declares
+      When query
+        """
+        SELECT reverse(CAST(id AS STRING)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
+
+    Scenario: a nullable column input to reverse stays nullable
+      When query
+        """
+        SELECT reverse(c) AS result FROM VALUES ('Spark SQL'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/collection/size.feature b/python/pysail/tests/spark/function/features/collection/size.feature
new file mode 100644
index 000000000..da8456699
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/collection/size.feature
@@ -0,0 +1,27 @@
+Feature: size output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to size yields the schema Spark declares
+      When query
+        """
+        SELECT size(array('b', 'd', 'c', 'a')) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to size stays nullable
+      When query
+        """
+        SELECT size(c) AS result FROM VALUES (array('b', 'd', 'c', 'a')), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/collection/try_element_at.feature b/python/pysail/tests/spark/function/features/collection/try_element_at.feature
new file mode 100644
index 000000000..d671159df
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/collection/try_element_at.feature
@@ -0,0 +1,27 @@
+Feature: try_element_at output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to try_element_at yields the schema Spark declares
+      When query
+        """
+        SELECT try_element_at(array(1, 2, 3), 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a nullable column input to try_element_at stays nullable
+      When query
+        """
+        SELECT try_element_at(c, 2) AS result FROM VALUES (array(1, 2, 3)), (CAST(NULL AS ARRAY)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/concat.feature b/python/pysail/tests/spark/function/features/concat.feature
deleted file mode 100644
index 1c04619b1..000000000
--- a/python/pysail/tests/spark/function/features/concat.feature
+++ /dev/null
@@ -1,688 +0,0 @@
-Feature: concat function
-
-  Rule: Basic concatenation
-
-    Scenario: concat two integer arrays
-      When query
-        """
-        SELECT concat(array(1, 2, 3), array(4, 5)) AS result
-        """
-      Then query result
-        | result          |
-        | [1, 2, 3, 4, 5] |
-
-    Scenario: concat two string arrays
-      When query
-        """
-        SELECT concat(array('a', 'b'), array('c')) AS result
-        """
-      Then query result
-        | result    |
-        | [a, b, c] |
-
-  Rule: Empty array handling
-
-    Scenario: concat empty array with typed array
-      When query
-        """
-        SELECT concat(array(), array(1, 2, 3)) AS result
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-
-    Scenario: concat typed array with empty array
-      When query
-        """
-        SELECT concat(array(1, 2), array()) AS result
-        """
-      Then query result
-        | result |
-        | [1, 2] |
-
-  Rule: Null propagation
-
-    Scenario: concat array with null returns null
-      When query
-        """
-        SELECT concat(array(1, 2), CAST(NULL AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: concat null with array returns null
-      When query
-        """
-        SELECT concat(CAST(NULL AS ARRAY), array(1, 2)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: String concatenation
-
-    Scenario: basic string concatenation
-      When query
-        """
-        SELECT concat('Spark', 'SQL') AS result
-        """
-      Then query result
-        | result   |
-        | SparkSQL |
-
-    Scenario: three string arguments
-      When query
-        """
-        SELECT concat('Hello', ', ', 'World') AS result
-        """
-      Then query result
-        | result        |
-        | Hello, World  |
-
-    Scenario: single string argument
-      When query
-        """
-        SELECT concat('hello') AS result
-        """
-      Then query result
-        | result |
-        | hello  |
-
-    Scenario: zero arguments returns empty string
-      When query
-        """
-        SELECT concat() AS result
-        """
-      Then query result
-        | result |
-        |        |
-
-    Scenario: empty strings
-      When query
-        """
-        SELECT concat('', '') AS result
-        """
-      Then query result
-        | result |
-        |        |
-
-    Scenario: empty string with non-empty string
-      When query
-        """
-        SELECT concat('', 'hello') AS result
-        """
-      Then query result
-        | result |
-        | hello  |
-
-    Scenario: whitespace strings
-      When query
-        """
-        SELECT concat('  ', '  ') AS result
-        """
-      Then query result
-        | result |
-        |        |
-
-    Scenario: many string arguments
-      When query
-        """
-        SELECT concat('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h') AS result
-        """
-      Then query result
-        | result   |
-        | abcdefgh |
-
-  Rule: String NULL propagation
-
-    Scenario: NULL only returns NULL
-      When query
-        """
-        SELECT concat(NULL) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: NULL with string returns NULL
-      When query
-        """
-        SELECT concat(NULL, 'b') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: string with NULL returns NULL
-      When query
-        """
-        SELECT concat('a', NULL) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: NULL and NULL returns NULL
-      When query
-        """
-        SELECT concat(NULL, NULL) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: string NULL string returns NULL
-      When query
-        """
-        SELECT concat('a', NULL, 'b') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: typed NULL string returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS STRING)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: typed NULL with string returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS STRING), 'b') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Type coercion to string
-
-    Scenario: TINYINT coerced to string
-      When query
-        """
-        SELECT concat(CAST(1 AS TINYINT)) AS result
-        """
-      Then query result
-        | result |
-        | 1      |
-
-    Scenario: INT coerced to string
-      When query
-        """
-        SELECT concat(1) AS result
-        """
-      Then query result
-        | result |
-        | 1      |
-
-    Scenario: INT and INT concatenated as strings
-      When query
-        """
-        SELECT concat(1, 2) AS result
-        """
-      Then query result
-        | result |
-        | 12     |
-
-    Scenario: DOUBLE coerced to string
-      When query
-        """
-        SELECT concat(1.0) AS result
-        """
-      Then query result
-        | result |
-        | 1.0    |
-
-    Scenario: DECIMAL coerced to string
-      When query
-        """
-        SELECT concat(CAST(1.0 AS DECIMAL(10,2))) AS result
-        """
-      Then query result
-        | result |
-        | 1.00   |
-
-    Scenario: BOOLEAN coerced to string
-      When query
-        """
-        SELECT concat(true) AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: DATE coerced to string
-      When query
-        """
-        SELECT concat(DATE '2024-01-15') AS result
-        """
-      Then query result
-        | result     |
-        | 2024-01-15 |
-
-    Scenario: TIMESTAMP coerced to string
-      When query
-        """
-        SELECT concat(TIMESTAMP '2024-01-15 12:00:00') AS result
-        """
-      Then query result
-        | result              |
-        | 2024-01-15 12:00:00 |
-
-    Scenario: string with INT coercion
-      When query
-        """
-        SELECT concat('hello', 1) AS result
-        """
-      Then query result
-        | result |
-        | hello1 |
-
-  Rule: Binary concatenation
-
-    Scenario: basic binary concatenation
-      When query
-        """
-        SELECT concat(X'4865', X'6C6C6F') AS result
-        """
-      Then query result
-        | result           |
-        | [48 65 6C 6C 6F] |
-
-    Scenario: single binary argument
-      When query
-        """
-        SELECT concat(X'48656C6C6F') AS result
-        """
-      Then query result
-        | result           |
-        | [48 65 6C 6C 6F] |
-
-    Scenario: three binary arguments
-      When query
-        """
-        SELECT concat(X'48', X'65', X'6C6C6F') AS result
-        """
-      Then query result
-        | result           |
-        | [48 65 6C 6C 6F] |
-
-    Scenario: empty binary with binary
-      When query
-        """
-        SELECT concat(X'', X'4865') AS result
-        """
-      Then query result
-        | result  |
-        | [48 65] |
-
-    Scenario: binary with string coercion
-      When query
-        """
-        SELECT concat(X'48656C6C6F', 'world') AS result
-        """
-      Then query result
-        | result     |
-        | Helloworld |
-
-    Scenario: string with binary coercion
-      When query
-        """
-        SELECT concat('hello', X'48656C6C6F') AS result
-        """
-      Then query result
-        | result     |
-        | helloHello |
-
-  Rule: Binary NULL propagation
-
-    Scenario: NULL typed binary returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS BINARY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: NULL binary with binary returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS BINARY), X'48') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: binary with NULL binary returns NULL
-      When query
-        """
-        SELECT concat(X'48', CAST(NULL AS BINARY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: two NULL binaries returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS BINARY), CAST(NULL AS BINARY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Array concatenation
-
-    Scenario: concat two boolean arrays
-      When query
-        """
-        SELECT concat(array(true), array(false)) AS result
-        """
-      Then query result
-        | result        |
-        | [true, false] |
-
-    Scenario: concat nested arrays
-      When query
-        """
-        SELECT concat(array(array(1, 2)), array(array(3, 4))) AS result
-        """
-      Then query result
-        | result           |
-        | [[1, 2], [3, 4]] |
-
-    Scenario: concat array with null elements
-      When query
-        """
-        SELECT concat(array(1, NULL, 3), array(4)) AS result
-        """
-      Then query result
-        | result          |
-        | [1, NULL, 3, 4] |
-
-    Scenario: concat three arrays
-      When query
-        """
-        SELECT concat(array(1, 2, 3), array(4, 5), array(6)) AS result
-        """
-      Then query result
-        | result             |
-        | [1, 2, 3, 4, 5, 6] |
-
-    Scenario: concat two empty arrays
-      When query
-        """
-        SELECT concat(array(), array()) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: NULL typed array returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: typed NULL array with typed NULL array returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS ARRAY), CAST(NULL AS ARRAY)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Multi-row behavior
-
-    Scenario: string concat from table with NULLs
-      When query
-        """
-        SELECT concat(a, b) AS result
-        FROM VALUES ('hello', ' world'), (NULL, 'x'), ('a', NULL) AS t(a, b)
-        """
-      Then query result
-        | result      |
-        | hello world |
-        | NULL        |
-        | NULL        |
-
-    Scenario: array concat from table with NULLs
-      When query
-        """
-        SELECT concat(a, b) AS result
-        FROM VALUES (array(1, 2), array(3)), (NULL, array(4)), (array(5), NULL) AS t(a, b)
-        """
-      Then query result
-        | result    |
-        | [1, 2, 3] |
-        | NULL      |
-        | NULL      |
-
-  Rule: Error cases
-
-    Scenario: MAP type is rejected
-      When query
-        """
-        SELECT concat(map('a', 1)) AS result
-        """
-      Then query error .*
-
-    Scenario: STRUCT type is rejected
-      When query
-        """
-        SELECT concat(named_struct('a', 1)) AS result
-        """
-      Then query error .*
-
-    Scenario: untyped NULL mixed with typed array is rejected
-      When query
-        """
-        SELECT concat(NULL, array(1, 2)) AS result
-        """
-      Then query error .*
-
-    Scenario: typed array mixed with untyped NULL is rejected
-      When query
-        """
-        SELECT concat(array(1, 2), NULL) AS result
-        """
-      Then query error .*
-
-  Rule: Timestamp coercion to string
-
-    Scenario: TIMESTAMP coerced to string with microseconds
-      When query
-        """
-        SELECT concat(TIMESTAMP '2024-01-15 12:00:00.123456') AS result
-        """
-      Then query result
-        | result                     |
-        | 2024-01-15 12:00:00.123456 |
-
-    Scenario: TIMESTAMP concatenated with string suffix
-      When query
-        """
-        SELECT concat(TIMESTAMP '2024-01-15 12:00:00', '_suffix') AS result
-        """
-      Then query result
-        | result                      |
-        | 2024-01-15 12:00:00_suffix  |
-
-    Scenario: string prefix concatenated with TIMESTAMP
-      When query
-        """
-        SELECT concat('prefix_', TIMESTAMP '2024-01-15 12:00:00') AS result
-        """
-      Then query result
-        | result                      |
-        | prefix_2024-01-15 12:00:00  |
-
-    Scenario: two TIMESTAMPs cast to string concatenated with separator
-      When query
-        """
-        SELECT concat(
-          CAST(TIMESTAMP '2024-01-15 12:00:00' AS STRING),
-          '_',
-          CAST(TIMESTAMP '2024-01-16 13:14:15' AS STRING)
-        ) AS result
-        """
-      Then query result
-        | result                                    |
-        | 2024-01-15 12:00:00_2024-01-16 13:14:15  |
-
-    Scenario: TIMESTAMP NULL with string returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS TIMESTAMP), 'x') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: TIMESTAMP_NTZ coerced to string
-      When query
-        """
-        SELECT concat(TIMESTAMP_NTZ '2024-01-15 12:00:00') AS result
-        """
-      Then query result
-        | result              |
-        | 2024-01-15 12:00:00 |
-
-    Scenario: TIMESTAMP_NTZ coerced to string with microseconds
-      When query
-        """
-        SELECT concat(TIMESTAMP_NTZ '2024-01-15 12:00:00.123456') AS result
-        """
-      Then query result
-        | result                     |
-        | 2024-01-15 12:00:00.123456 |
-
-    Scenario: TIMESTAMP_NTZ concatenated with string suffix
-      When query
-        """
-        SELECT concat(TIMESTAMP_NTZ '2024-01-15 12:00:00', '_end') AS result
-        """
-      Then query result
-        | result                  |
-        | 2024-01-15 12:00:00_end |
-
-    Scenario: TIMESTAMP_NTZ NULL with string returns NULL
-      When query
-        """
-        SELECT concat(CAST(NULL AS TIMESTAMP_NTZ), 'x') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-  Rule: Nested concat
-
-    Scenario: nested concat with string constants folds to single value
-      When query
-        """
-        SELECT concat(concat('a', 'b'), 'c') AS result
-        """
-      Then query result
-        | result |
-        | abc    |
-
-    Scenario: nested concat with column does not flatten but returns correct result
-      When query
-        """
-        SELECT concat(concat(v, 'b'), 'c') AS result
-        FROM VALUES ('a') AS t(v)
-        """
-      Then query result
-        | result |
-        | abc    |
-
-  Rule: Single-argument simplify edge cases
-
-    Scenario: single timestamp column coerced to string
-      When query
-        """
-        SELECT concat(ts) AS result
-        FROM VALUES (TIMESTAMP '2024-01-15 12:00:00') AS t(ts)
-        """
-      Then query result
-        | result              |
-        | 2024-01-15 12:00:00 |
-
-    Scenario: single timestamp column with microseconds
-      When query
-        """
-        SELECT concat(ts) AS result
-        FROM VALUES (TIMESTAMP '2024-01-15 12:00:00.123456') AS t(ts)
-        """
-      Then query result
-        | result                     |
-        | 2024-01-15 12:00:00.123456 |
-
-    Scenario: single timestamp column explicit cast matches bare column
-      When query
-        """
-        SELECT concat(CAST(ts AS STRING)) AS result
-        FROM VALUES (TIMESTAMP '2024-01-15 12:00:00') AS t(ts)
-        """
-      Then query result
-        | result              |
-        | 2024-01-15 12:00:00 |
-
-    Scenario: single INT column coerced to string
-      When query
-        """
-        SELECT concat(n) AS result
-        FROM VALUES (42) AS t(n)
-        """
-      Then query result
-        | result |
-        | 42     |
-
-    Scenario: single DOUBLE column coerced to string
-      When query
-        """
-        SELECT concat(d) AS result
-        FROM VALUES (3.14) AS t(d)
-        """
-      Then query result
-        | result |
-        | 3.14   |
-
-    Scenario: single BOOLEAN column coerced to string
-      When query
-        """
-        SELECT concat(b) AS result
-        FROM VALUES (true) AS t(b)
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: single NULL timestamp column returns NULL
-      When query
-        """
-        SELECT concat(ts) AS result
-        FROM VALUES (CAST(NULL AS TIMESTAMP)) AS t(ts)
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: single TIMESTAMP_NTZ column coerced to string
-      When query
-        """
-        SELECT concat(ts) AS result
-        FROM VALUES (TIMESTAMP_NTZ '2024-01-15 12:00:00') AS t(ts)
-        """
-      Then query result
-        | result              |
-        | 2024-01-15 12:00:00 |
diff --git a/python/pysail/tests/spark/function/features/concat_simplify.feature b/python/pysail/tests/spark/function/features/concat_simplify.feature
deleted file mode 100644
index 9243159a2..000000000
--- a/python/pysail/tests/spark/function/features/concat_simplify.feature
+++ /dev/null
@@ -1,161 +0,0 @@
-Feature: concat() — simplify hook (single-argument identity)
-
-  Rule: String, array, and binary inputs — identity, no coercion needed
-
-    Scenario: concat of single string literal is identity
-      When query
-        """
-        SELECT concat('hello') AS result
-        """
-      Then query result
-        | result |
-        | hello  |
-
-    Scenario: concat of single string column propagates values and nulls
-      When query
-        """
-        SELECT concat(v) AS result FROM VALUES
-          ('hello'),
-          ('world'),
-          (NULL)
-        AS t(v) ORDER BY result
-        """
-      Then query result ordered
-        | result |
-        | NULL   |
-        | hello  |
-        | world  |
-
-    Scenario: concat of single array column is identity
-      When query
-        """
-        SELECT concat(v) AS result FROM VALUES
-          (array(1, 2, 3)),
-          (array(4, 5))
-        AS t(v) ORDER BY result
-        """
-      Then query result ordered
-        | result    |
-        | [1, 2, 3] |
-        | [4, 5]    |
-
-    Scenario: concat of single null string returns null
-      When query
-        """
-        SELECT concat(CAST(NULL AS STRING)) AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    # Regression coverage: `concat(array())` has arg type `List(Null)` which the
-    # simplify hook matches. `return_type` for `[List(Null)]` is also `List(Null)`,
-    # so simplify is type-preserving here — both the simplify and invoke paths
-    # produce an empty list.
-    Scenario: concat of single empty array literal returns empty array
-      When query
-        """
-        SELECT concat(array()) AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-    Scenario: concat of single binary column is identity
-      When query
-        """
-        SELECT concat(v) AS result FROM VALUES
-          (CAST('hello' AS BINARY)),
-          (CAST('world' AS BINARY)),
-          (CAST(NULL AS BINARY))
-        AS t(v) ORDER BY result
-        """
-      Then query result ordered
-        | result            |
-        | NULL              |
-        | [68 65 6C 6C 6F]  |
-        | [77 6F 72 6C 64]  |
-
-  Rule: Non-string inputs — coerced to string, not simplified away
-
-    Scenario: concat of integer literal coerces to string
-      When query
-        """
-        SELECT concat(1) AS result
-        """
-      Then query result
-        | result |
-        | 1      |
-
-    Scenario: concat of boolean literal coerces to string
-      When query
-        """
-        SELECT concat(true) AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: concat of integer column coerces to string and propagates nulls
-      When query
-        """
-        SELECT concat(v) AS result FROM VALUES
-          (CAST(1 AS INT)),
-          (CAST(2 AS INT)),
-          (CAST(NULL AS INT))
-        AS t(v) ORDER BY result
-        """
-      Then query result ordered
-        | result |
-        | NULL   |
-        | 1      |
-        | 2      |
-
-    # Regression coverage: timestamps must NOT be matched by the simplify hook
-    # because `invoke_with_args` applies Spark-specific timestamp formatting
-    # (`spark_format_timestamp_str`) that returns `YYYY-MM-DD HH:MM:SS` without
-    # the Arrow `T` separator or timezone suffix. Simplifying `concat(ts)` to
-    # `ts` would leak Arrow's ISO 8601 rendering instead.
-    Scenario: concat of single timestamp coerces to Spark-formatted string
-      When query
-        """
-        SELECT concat(CAST('2024-01-15 12:00:00' AS TIMESTAMP)) AS result
-        """
-      Then query result
-        | result              |
-        | 2024-01-15 12:00:00 |
-
-  Rule: Plan snapshots — simplify removes UDF call only for single-argument identity cases (string/array/binary), and keeps it for coercion or multi-arg array concat
-
-    @sail-only
-    Scenario: EXPLAIN concat of single string column — no spark_concat in plan
-      When query
-        """
-        EXPLAIN SELECT concat(v) AS result FROM VALUES
-          ('hello'),
-          ('world')
-        AS t(v)
-        """
-      Then query plan matches snapshot
-
-    @sail-only
-    Scenario: EXPLAIN concat of two array columns — spark_concat stays in plan
-      When query
-        """
-        EXPLAIN SELECT concat(a, b) AS result FROM VALUES
-          (array(1, 2), array(3, 4)),
-          (array(5), array(6, 7))
-        AS t(a, b)
-        """
-      Then query plan matches snapshot
-
-    @sail-only
-    Scenario: EXPLAIN concat of single integer column — spark_concat stays in plan
-      When query
-        """
-        EXPLAIN SELECT concat(v) AS result FROM VALUES
-          (CAST(1 AS INT)),
-          (CAST(2 AS INT))
-        AS t(v)
-        """
-      Then query plan matches snapshot
diff --git a/python/pysail/tests/spark/function/features/concat_ws.feature b/python/pysail/tests/spark/function/features/concat_ws.feature
deleted file mode 100644
index 557c98855..000000000
--- a/python/pysail/tests/spark/function/features/concat_ws.feature
+++ /dev/null
@@ -1,265 +0,0 @@
-Feature: concat_ws function
-
-  Rule: concat_ws with scalar arguments
-
-    Scenario: concat_ws with multiple string arguments
-      When query
-        """
-        SELECT concat_ws(',', 'a', 'b', 'c') AS result
-        """
-      Then query result
-        | result  |
-        | a,b,c   |
-
-    Scenario: concat_ws with null arguments
-      When query
-        """
-        SELECT concat_ws(',', 'a', NULL, 'c') AS result
-        """
-      Then query result
-        | result  |
-        | a,c     |
-
-    Scenario: concat_ws with single argument
-      When query
-        """
-        SELECT concat_ws(',', 'a') AS result
-        """
-      Then query result
-        | result  |
-        | a       |
-
-    Scenario: concat_ws with no arguments after separator
-      When query
-        """
-        SELECT concat_ws(',') AS result
-        """
-      Then query result
-        | result  |
-        |         |
-
-    Scenario: concat_ws with null separator returns null
-      When query
-        """
-        SELECT concat_ws(NULL, 'a', 'b', 'c') AS result
-        """
-      Then query result
-        | result  |
-        | NULL    |
-
-    Scenario: concat_ws coerces integer arguments to string
-      When query
-        """
-        SELECT concat_ws('-', 'a', 1, 2) AS result
-        """
-      Then query result
-        | result  |
-        | a-1-2   |
-
-    Scenario: concat_ws coerces double arguments to string
-      When query
-        """
-        SELECT concat_ws('-', 'a', 1.5) AS result
-        """
-      Then query result
-        | result  |
-        | a-1.5   |
-
-    Scenario: concat_ws coerces boolean arguments to string
-      When query
-        """
-        SELECT concat_ws('-', 'a', true, false) AS result
-        """
-      Then query result
-        | result         |
-        | a-true-false   |
-
-  Rule: concat_ws with array arguments
-
-    Scenario: concat_ws with array argument
-      When query
-        """
-        SELECT concat_ws(',', array('a', 'b', 'c')) AS result
-        """
-      Then query result
-        | result  |
-        | a,b,c   |
-
-    Scenario: concat_ws with array containing nulls
-      When query
-        """
-        SELECT concat_ws(',', array('a', NULL, 'c')) AS result
-        """
-      Then query result
-        | result  |
-        | a,c     |
-
-    Scenario: concat_ws with multiple arrays
-      When query
-        """
-        SELECT concat_ws(',', array('a', 'b'), array('c', 'd')) AS result
-        """
-      Then query result
-        | result    |
-        | a,b,c,d   |
-
-    Scenario: concat_ws with mixed scalar and array arguments
-      When query
-        """
-        SELECT concat_ws(',', 'x', array('a', 'b'), 'y') AS result
-        """
-      Then query result
-        | result    |
-        | x,a,b,y   |
-
-  Rule: concat_ws over multiple rows (column inputs)
-
-    Scenario: concat_ws null separator over a column returns NULL per row
-      When query
-        """
-        SELECT concat_ws(NULL, v) AS result
-        FROM VALUES ('a'), ('b'), ('c') AS t(v)
-        ORDER BY v
-        """
-      Then query result ordered
-        | result |
-        | NULL   |
-        | NULL   |
-        | NULL   |
-
-    Scenario: concat_ws skips per-row NULL over a column
-      When query
-        """
-        SELECT concat_ws(',', v, 'X') AS result
-        FROM VALUES ('a'), (CAST(NULL AS STRING)), ('c') AS t(v)
-        ORDER BY v NULLS FIRST
-        """
-      Then query result ordered
-        | result |
-        | X      |
-        | a,X    |
-        | c,X    |
-
-    Scenario: concat_ws with a per-row separator column including NULL
-      When query
-        """
-        SELECT concat_ws(sep, a, b) AS result FROM VALUES
-          (0, ',', 'a', 'b'),
-          (1, CAST(NULL AS STRING), 'a', 'b'),
-          (2, '|', 'x', 'y')
-        AS t(id, sep, a, b) ORDER BY id
-        """
-      Then query result ordered
-        | result |
-        | a,b    |
-        | NULL   |
-        | x\|y   |
-
-    Scenario: concat_ws with all-scalar arguments broadcasts over rows
-      When query
-        """
-        SELECT concat_ws(',', 'a', 'b') AS result
-        FROM VALUES (1), (2), (3) AS t(x)
-        """
-      Then query result
-        | result |
-        | a,b    |
-        | a,b    |
-        | a,b    |
-
-  Rule: concat_ws argument coercion and validation
-
-    Scenario: concat_ws skips a whole-NULL array argument
-      When query
-        """
-        SELECT concat_ws(',', CAST(NULL AS ARRAY), 'z') AS result
-        """
-      Then query result
-        | result |
-        | z      |
-
-    Scenario: concat_ws coerces binary to string
-      When query
-        """
-        SELECT concat_ws(',', X'4869') AS result
-        """
-      Then query result
-        | result |
-        | Hi     |
-
-    Scenario: concat_ws rejects struct arguments
-      When query
-        """
-        SELECT concat_ws(',', named_struct('a', 1)) AS result
-        """
-      Then query error .*
-
-    Scenario: concat_ws with zero arguments errors
-      When query
-        """
-        SELECT concat_ws() AS result
-        """
-      Then query error .*
-
-    Scenario: concat_ws does not skip empty-string arguments (only NULLs)
-      When query
-        """
-        SELECT concat_ws(',', '', 'a', '', 'b') AS result
-        """
-      Then query result
-        | result |
-        | ,a,,b  |
-
-    Scenario: concat_ws with all-NULL arguments returns empty string
-      When query
-        """
-        SELECT concat_ws(',', CAST(NULL AS STRING), CAST(NULL AS STRING)) AS result
-        """
-      Then query result
-        | result |
-        |        |
-
-    Scenario: concat_ws with empty separator concatenates with nothing between
-      When query
-        """
-        SELECT concat_ws('', 'a', 'b', 'c') AS result
-        """
-      Then query result
-        | result |
-        | abc    |
-
-    Scenario: concat_ws with an all-NULL array returns empty string
-      When query
-        """
-        SELECT concat_ws(',', array(CAST(NULL AS STRING), CAST(NULL AS STRING))) AS result
-        """
-      Then query result
-        | result |
-        |        |
-
-    Scenario: concat_ws coerces a numeric separator to string
-      When query
-        """
-        SELECT concat_ws(1, 'a', 'b') AS result
-        """
-      Then query result
-        | result |
-        | a1b    |
-
-    Scenario: concat_ws nested inside concat_ws
-      When query
-        """
-        SELECT concat_ws('|', concat_ws(',', 'a', 'b'), concat_ws(',', 'c', 'd')) AS result
-        """
-      Then query result
-        | result   |
-        | a,b\|c,d |
-
-    Scenario: concat_ws with an empty array returns empty string
-      When query
-        """
-        SELECT concat_ws(',', array()) AS result
-        """
-      Then query result
-        | result |
-        |        |
diff --git a/python/pysail/tests/spark/function/features/conditional/coalesce.feature b/python/pysail/tests/spark/function/features/conditional/coalesce.feature
new file mode 100644
index 000000000..5daeb28db
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/coalesce.feature
@@ -0,0 +1,95 @@
+Feature: coalesce returns the first non-null argument
+
+  Rule: Spark-compatible coercion for mixed string and temporal arguments
+
+    Scenario Outline: Coercion: 
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        WITH t() AS (
+          SELECT 
+        )
+        SELECT
+          coalesce() AS result,
+          typeof(coalesce()) AS result_type
+        FROM t
+        """
+      Then query result
+        | result   | result_type |
+        |  | string      |
+
+      Examples:
+        | case                                                                               | cols                      | vals                                                  | args                      | result              |
+        | Coalesce null string column falls back to date column as string                    | string_col, date_col      | CAST(NULL AS STRING), DATE '2024-01-15'               | string_col, date_col      | 2024-01-15          |
+        | Coalesce null string column falls back to timestamp column as string               | string_col, timestamp_col | CAST(NULL AS STRING), TIMESTAMP '2024-01-15 10:30:00' | string_col, timestamp_col | 2024-01-15 10:30:00 |
+        | Coalesce string literal before a date column wins without temporal casting         | date_col                  | DATE '2024-01-15'                                     | 'default', date_col       | default             |
+        | Coalesce non-null string column before a date column wins without temporal casting | string_col, date_col      | CAST('hello' AS STRING), DATE '2024-01-15'            | string_col, date_col      | hello               |
+        | Coalesce date column before a string column returns a string value                 | date_col, string_col      | DATE '2024-01-15', CAST('fallback' AS STRING)         | date_col, string_col      | 2024-01-15          |
+
+    Scenario: Coalesce multiple mixed temporal arguments still coerces to string
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT
+          coalesce(
+            CAST(NULL AS STRING),
+            CAST(NULL AS DATE),
+            TIMESTAMP '2024-01-15 10:30:00'
+          ) AS result,
+          typeof(coalesce(
+            CAST(NULL AS STRING),
+            CAST(NULL AS DATE),
+            TIMESTAMP '2024-01-15 10:30:00'
+          )) AS result_type
+        """
+      Then query result
+        | result              | result_type |
+        | 2024-01-15 10:30:00 | string      |
+
+    Scenario: Coalesce all-null mixed string and date arguments returns null with string type
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT
+          coalesce(CAST(NULL AS STRING), CAST(NULL AS DATE)) AS result,
+          typeof(coalesce(CAST(NULL AS STRING), CAST(NULL AS DATE))) AS result_type
+        """
+      Then query result
+        | result | result_type |
+        | NULL   | string      |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null argument yields a non-nullable value
+      When query
+        """
+        SELECT coalesce(1, 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+    Scenario: a non-null column argument yields a non-nullable value
+      When query
+        """
+        SELECT coalesce(id, 0) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = false)
+        """
+
+    Scenario: all-nullable arguments stay nullable
+      When query
+        """
+        SELECT coalesce(c, d) AS result FROM VALUES (CAST(NULL AS INT), CAST(NULL AS INT)) AS t(c, d)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/if.feature b/python/pysail/tests/spark/function/features/conditional/if.feature
new file mode 100644
index 000000000..0397ac4e3
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/if.feature
@@ -0,0 +1,26 @@
+Feature: if output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to if yields the schema Spark declares
+      When query
+        """
+        SELECT if(1 < 2, 'a', 'b') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
+
+    Scenario: a nullable column input to if stays nullable
+      When query
+        """
+        SELECT if(c, 'a', 'b') AS result FROM VALUES (1 < 2), (CAST(NULL AS BOOLEAN)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/ifnull.feature b/python/pysail/tests/spark/function/features/conditional/ifnull.feature
new file mode 100644
index 000000000..23a7cbdd1
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/ifnull.feature
@@ -0,0 +1,17 @@
+Feature: ifnull output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to ifnull yields the schema Spark declares
+      When query
+        """
+        SELECT ifnull(NULL, array('2')) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/nanvl.feature b/python/pysail/tests/spark/function/features/conditional/nanvl.feature
new file mode 100644
index 000000000..803aa71f0
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/nanvl.feature
@@ -0,0 +1,38 @@
+Feature: nanvl output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to nanvl yields the schema Spark declares
+      When query
+        """
+        SELECT nanvl(cast('NaN' as double), 123) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: double (nullable = true)
+        """
+
+    @sail-bug
+    Scenario: a non-null column input to nanvl yields the schema Spark declares
+      When query
+        """
+        SELECT nanvl(CAST(id AS DOUBLE), 123) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: double (nullable = false)
+        """
+
+    Scenario: a nullable column input to nanvl stays nullable
+      When query
+        """
+        SELECT nanvl(c, 123) AS result FROM VALUES (cast('NaN' as double)), (CAST(NULL AS DOUBLE)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: double (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/nullif.feature b/python/pysail/tests/spark/function/features/conditional/nullif.feature
new file mode 100644
index 000000000..ce97c638e
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/nullif.feature
@@ -0,0 +1,37 @@
+Feature: nullif output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to nullif yields the schema Spark declares
+      When query
+        """
+        SELECT nullif(2, 2) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a non-null column input to nullif yields the schema Spark declares
+      When query
+        """
+        SELECT nullif(CAST(id AS INT), 2) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a nullable column input to nullif stays nullable
+      When query
+        """
+        SELECT nullif(c, 2) AS result FROM VALUES (2), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/nullifzero.feature b/python/pysail/tests/spark/function/features/conditional/nullifzero.feature
new file mode 100644
index 000000000..1e8e5ae28
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/nullifzero.feature
@@ -0,0 +1,75 @@
+Feature: nullifzero output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to nullifzero yields the schema Spark declares
+      When query
+        """
+        SELECT nullifzero(0) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a non-null column input to nullifzero yields the schema Spark declares
+      When query
+        """
+        SELECT nullifzero(CAST(id AS INT)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a nullable column input to nullifzero stays nullable
+      When query
+        """
+        SELECT nullifzero(c) AS result FROM VALUES (0), (CAST(NULL AS INT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+  Rule: Result values (migrated from test_nullifzero.txt doctests)
+
+    Scenario Outline: Result values: 
+      When query
+        """
+        SELECT nullifzero(), typeof(nullifzero())
+        """
+      Then query result
+        | nullifzero() | typeof(nullifzero()) |
+        |            |                     |
+
+      Examples:
+        | case                            | arg                         | value | type          |
+        | nullifzero doctest #1 (result)  | 0                           | NULL  | int           |
+        | nullifzero doctest #2 (result)  | 2                           | 2     | int           |
+        | nullifzero doctest #3 (result)  | CAST(0 AS TINYINT)          | NULL  | tinyint       |
+        | nullifzero doctest #4 (result)  | CAST(2 AS TINYINT)          | 2     | tinyint       |
+        | nullifzero doctest #5 (result)  | CAST(0 AS SMALLINT)         | NULL  | smallint      |
+        | nullifzero doctest #6 (result)  | CAST(0 AS BIGINT)           | NULL  | bigint        |
+        | nullifzero doctest #7 (result)  | CAST(0.0 AS FLOAT)          | NULL  | float         |
+        | nullifzero doctest #8 (result)  | CAST(1.5 AS FLOAT)          | 1.5   | float         |
+        | nullifzero doctest #9 (result)  | CAST(0.0 AS DOUBLE)         | NULL  | double        |
+        | nullifzero doctest #10 (result) | CAST(2.5 AS DOUBLE)         | 2.5   | double        |
+        | nullifzero doctest #11 (result) | CAST(0.00 AS DECIMAL(10,2)) | NULL  | decimal(10,2) |
+        | nullifzero doctest #12 (result) | CAST(5.25 AS DECIMAL(10,2)) | 5.25  | decimal(10,2) |
+        | nullifzero doctest #14 (result) | NULL                        | NULL  | int           |
+
+    # Kept separate: Spark derives the column name as `nullifzero((- 1))`, not
+    # `nullifzero(-1)`, so the SQL argument and the header cannot share one slot.
+    Scenario: nullifzero doctest #13 (result)
+      When query
+        """
+        SELECT nullifzero(-1), typeof(nullifzero(-1))
+        """
+      Then query result
+        | nullifzero((- 1)) | typeof(nullifzero((- 1))) |
+        | -1                | int                       |
diff --git a/python/pysail/tests/spark/function/features/conditional/nvl.feature b/python/pysail/tests/spark/function/features/conditional/nvl.feature
new file mode 100644
index 000000000..af87623ae
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/nvl.feature
@@ -0,0 +1,17 @@
+Feature: nvl output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to nvl yields the schema Spark declares
+      When query
+        """
+        SELECT nvl(NULL, array('2')) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = false)
+         |    |-- element: string (containsNull = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/nvl2.feature b/python/pysail/tests/spark/function/features/conditional/nvl2.feature
new file mode 100644
index 000000000..60e411909
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/nvl2.feature
@@ -0,0 +1,15 @@
+Feature: nvl2 output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to nvl2 yields the schema Spark declares
+      When query
+        """
+        SELECT nvl2(NULL, 2, 1) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/when.feature b/python/pysail/tests/spark/function/features/conditional/when.feature
new file mode 100644
index 000000000..fa46e92bf
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/when.feature
@@ -0,0 +1,16 @@
+Feature: when output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to when yields the schema Spark declares
+      When query
+        """
+        SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: decimal(11,1) (nullable = false)
+        """
diff --git a/python/pysail/tests/spark/function/features/conditional/zeroifnull.feature b/python/pysail/tests/spark/function/features/conditional/zeroifnull.feature
new file mode 100644
index 000000000..5becf05e0
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conditional/zeroifnull.feature
@@ -0,0 +1,53 @@
+Feature: zeroifnull output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to zeroifnull yields the schema Spark declares
+      When query
+        """
+        SELECT zeroifnull(NULL) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = false)
+        """
+
+  Rule: Result values (migrated from test_zeroifnull.txt doctests)
+
+    Scenario Outline: Result values: 
+      When query
+        """
+        SELECT zeroifnull(), typeof(zeroifnull())
+        """
+      Then query result
+        | zeroifnull() | typeof(zeroifnull()) |
+        |            |                     |
+
+      Examples:
+        | case                            | arg                          | value | type     |
+        | zeroifnull doctest #1 (result)  | NULL                         | 0     | int      |
+        | zeroifnull doctest #2 (result)  | 2                            | 2     | int      |
+        | zeroifnull doctest #3 (result)  | CAST(NULL AS TINYINT)        | 0     | tinyint  |
+        | zeroifnull doctest #4 (result)  | CAST(5 AS TINYINT)           | 5     | tinyint  |
+        | zeroifnull doctest #5 (result)  | CAST(NULL AS SMALLINT)       | 0     | smallint |
+        | zeroifnull doctest #6 (result)  | CAST(NULL AS BIGINT)         | 0     | bigint   |
+        | zeroifnull doctest #7 (result)  | CAST(NULL AS FLOAT)          | 0.0   | float    |
+        | zeroifnull doctest #8 (result)  | CAST(3.14 AS FLOAT)          | 3.14  | float    |
+        | zeroifnull doctest #9 (result)  | CAST(NULL AS DOUBLE)         | 0.0   | double   |
+        | zeroifnull doctest #10 (result) | CAST(2.718 AS DOUBLE)        | 2.718 | double   |
+        | zeroifnull doctest #11 (result) | CAST(NULL AS DECIMAL(10,2))  | 0.0   | double   |
+        | zeroifnull doctest #12 (result) | CAST(42.99 AS DECIMAL(10,2)) | 42.99 | double   |
+        | zeroifnull doctest #13 (result) | 0                            | 0     | int      |
+
+    # Kept separate: Spark derives the column name as `zeroifnull((- 5))`, not
+    # `zeroifnull(-5)`, so the SQL argument and the header cannot share one slot.
+    Scenario: zeroifnull doctest #14 (result)
+      When query
+        """
+        SELECT zeroifnull(-5), typeof(zeroifnull(-5))
+        """
+      Then query result
+        | zeroifnull((- 5)) | typeof(zeroifnull((- 5))) |
+        | -5                | int                       |
diff --git a/python/pysail/tests/spark/function/features/conversion/cast.feature b/python/pysail/tests/spark/function/features/conversion/cast.feature
new file mode 100644
index 000000000..104ff9847
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conversion/cast.feature
@@ -0,0 +1,16 @@
+Feature: cast output schema
+
+  @function(nullability)
+  Rule: Output schema
+
+    @sail-bug
+    Scenario: a non-null literal input to cast yields the schema Spark declares
+      When query
+        """
+        SELECT cast('10' as int) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/conversion/cast_date_to_numeric.feature b/python/pysail/tests/spark/function/features/conversion/cast_date_to_numeric.feature
new file mode 100644
index 000000000..05eb400a8
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conversion/cast_date_to_numeric.feature
@@ -0,0 +1,46 @@
+Feature: CAST date to numeric types returns null
+
+  In Spark legacy mode, casting a DATE to any numeric type returns NULL.
+  In ANSI mode, the cast raises an error. TRY_CAST always returns NULL.
+
+  Rule: CAST date to numeric types returns null (legacy mode)
+
+    Scenario Outline: Legacy: 
+      Given config spark.sql.ansi.enabled = false
+      When query
+        """
+        SELECT CAST( AS ) AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+      Examples:
+        | case                               | value              | type          |
+        | cast date to int returns null      | DATE '2023-01-15'  | INT           |
+        | cast date to bigint returns null   | DATE '2023-01-15'  | BIGINT        |
+        | cast date to smallint returns null | DATE '2023-01-15'  | SMALLINT      |
+        | cast date to tinyint returns null  | DATE '2023-01-15'  | TINYINT       |
+        | cast date to float returns null    | DATE '2023-01-15'  | FLOAT         |
+        | cast date to double returns null   | DATE '2023-01-15'  | DOUBLE        |
+        | cast date to decimal returns null  | DATE '2023-01-15'  | DECIMAL(10,2) |
+        | cast date to boolean returns null  | DATE '2023-01-15'  | BOOLEAN       |
+        | cast null date to int returns null | CAST(NULL AS DATE) | INT           |
+
+  Rule: CAST date to numeric in ANSI mode raises error
+
+    @sail-only
+    Scenario Outline: ANSI: 
+      Given config spark.sql.ansi.enabled = true
+      When query
+        """
+        SELECT CAST(DATE '2023-01-15' AS ) AS result
+        """
+      Then query error cannot cast date
+      Given config spark.sql.ansi.enabled = false
+
+      Examples:
+        | case                                           | type    |
+        | cast date to int in ANSI mode raises error     | INT     |
+        | cast date to double in ANSI mode raises error  | DOUBLE  |
+        | cast date to boolean in ANSI mode raises error | BOOLEAN |
diff --git a/python/pysail/tests/spark/function/features/conversion/cast_nan_inf.feature b/python/pysail/tests/spark/function/features/conversion/cast_nan_inf.feature
new file mode 100644
index 000000000..56f7ececf
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/conversion/cast_nan_inf.feature
@@ -0,0 +1,161 @@
+Feature: CAST and type constructors with NaN and Infinity (issue #630)
+
+  Rule: FLOAT type constructor
+
+    Scenario Outline: FLOAT constructor: 
+      When query
+        """
+        SELECT FLOAT() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                     | arg         | result    |
+        | FLOAT NaN                | 'NAN'       | NaN       |
+        | FLOAT NaN lowercase      | 'nan'       | NaN       |
+        | FLOAT NaN mixed case     | 'Nan'       | NaN       |
+        | FLOAT negative NaN       | '-NaN'      | NaN       |
+        | FLOAT Infinity           | 'Infinity'  | Infinity  |
+        | FLOAT negative Infinity  | '-Infinity' | -Infinity |
+        | FLOAT Infinity lowercase | 'infinity'  | Infinity  |
+        | FLOAT normal value       | '42'        | 42.0      |
+
+    @sail-bug
+    # Sail does not trim spaces before parsing NaN
+    Scenario: FLOAT NaN with spaces
+      When query
+        """
+        SELECT FLOAT(' NaN ') AS result
+        """
+      Then query result
+        | result |
+        | NaN    |
+
+  Rule: DOUBLE type constructor
+
+    Scenario Outline: DOUBLE constructor: 
+      When query
+        """
+        SELECT DOUBLE() AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                               | arg         | result    |
+        | DOUBLE NaN                         | 'NAN'       | NaN       |
+        | DOUBLE Infinity uppercase          | 'INFINITY'  | Infinity  |
+        | DOUBLE negative Infinity uppercase | '-INFINITY' | -Infinity |
+        | DOUBLE normal value                | '3.14'      | 3.14      |
+
+    @sail-bug
+    # Sail does not trim spaces before parsing Infinity
+    Scenario: DOUBLE Infinity with spaces
+      When query
+        """
+        SELECT DOUBLE(' Infinity ') AS result
+        """
+      Then query result
+        | result   |
+        | Infinity |
+
+  Rule: CAST to FLOAT/DOUBLE
+
+    Scenario Outline: CAST: 
+      When query
+        """
+        SELECT CAST('NaN' AS ) AS result
+        """
+      Then query result
+        | result |
+        | NaN    |
+
+      Examples:
+        | case               | type   |
+        | CAST NaN to FLOAT  | FLOAT  |
+        | CAST NaN to DOUBLE | DOUBLE |
+
+  Rule: Integer types reject NaN and Infinity
+
+    Scenario Outline: Integer rejects: 
+      When query
+        """
+        SELECT  AS result
+        """
+      Then query error .*
+
+      Examples:
+        | case                      | expr                         |
+        | INT NaN errors            | INT('NAN')                   |
+        | CAST NaN to INT errors    | CAST('NaN' AS INT)           |
+        | INT Infinity errors       | INT('Infinity')              |
+        | BIGINT NaN errors         | BIGINT('NaN')                |
+        | SMALLINT NaN errors       | SMALLINT('NaN')              |
+        | TINYINT NaN errors        | TINYINT('NaN')               |
+        | DECIMAL NaN errors        | CAST('NaN' AS DECIMAL(10,2)) |
+        | INT invalid string errors | INT('hello')                 |
+
+  Rule: TRY_CAST with NaN
+
+    Scenario Outline: TRY_CAST: 
+      When query
+        """
+        SELECT TRY_CAST('NaN' AS ) AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                              | type  | result |
+        | TRY_CAST NaN to INT returns NULL  | INT   | NULL   |
+        | TRY_CAST NaN to FLOAT returns NaN | FLOAT | NaN    |
+
+  Rule: NaN arithmetic
+
+    Scenario Outline: NaN arithmetic: 
+      When query
+        """
+        SELECT  AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                   | expr                        | result |
+        | NaN plus number is NaN                 | FLOAT('NaN') + 1            | NaN    |
+        | NaN equals NaN is true in Spark        | FLOAT('NaN') = FLOAT('NaN') | true   |
+        | NaN greater than zero is true in Spark | FLOAT('NaN') > 0            | true   |
+
+  Rule: Multi-row with NaN and Infinity
+
+    Scenario: VALUES with FLOAT NaN Infinity and NULL
+      When query
+        """
+        SELECT * FROM VALUES (FLOAT('NaN')), (FLOAT('Infinity')), (FLOAT('-Infinity')), (NULL), (0.0), (1.5) AS t(v)
+        """
+      Then query result
+        | v         |
+        | NaN       |
+        | Infinity  |
+        | -Infinity |
+        | NULL      |
+        | 0.0       |
+        | 1.5       |
+
+    Scenario: VALUES with DOUBLE NaN Infinity and NULL
+      When query
+        """
+        SELECT * FROM VALUES (DOUBLE('NaN')), (DOUBLE('Infinity')), (DOUBLE('-Infinity')), (NULL), (0.0) AS t(v)
+        """
+      Then query result
+        | v         |
+        | NaN       |
+        | Infinity  |
+        | -Infinity |
+        | NULL      |
+        | 0.0       |
diff --git a/python/pysail/tests/spark/function/features/count_distinct_star.feature b/python/pysail/tests/spark/function/features/count_distinct_star.feature
deleted file mode 100644
index 5814e0663..000000000
--- a/python/pysail/tests/spark/function/features/count_distinct_star.feature
+++ /dev/null
@@ -1,96 +0,0 @@
-Feature: COUNT(DISTINCT *) function
-
-  Rule: COUNT(DISTINCT *) counts distinct rows
-
-    Scenario: count distinct star with duplicates
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, 1.0, 'a'), (2, 2.0, 'b'), (1, 1.0, 'a') AS t(a, b, c)
-        """
-      Then query result
-        | count(DISTINCT a, b, c) |
-        | 2                       |
-
-    Scenario: count distinct star all same
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, 1), (1, 1), (1, 1) AS t(a, b)
-        """
-      Then query result
-        | count(DISTINCT a, b) |
-        | 1                    |
-
-    Scenario: count distinct star all different
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, 1), (2, 2), (3, 3) AS t(a, b)
-        """
-      Then query result
-        | count(DISTINCT a, b) |
-        | 3                    |
-
-    Scenario: count distinct star single column
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1), (2), (1) AS t(a)
-        """
-      Then query result
-        | count(DISTINCT a) |
-        | 2                 |
-
-    Scenario: count distinct star with group by
-      When query
-        """
-        SELECT g, COUNT(DISTINCT *) FROM VALUES ('x', 1), ('x', 1), ('x', 2), ('y', 1) AS t(g, v) GROUP BY g ORDER BY g
-        """
-      Then query result ordered
-        | g | count(DISTINCT g, v) |
-        | x | 2                    |
-        | y | 1                    |
-
-    Scenario: count distinct star with mixed case column names
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, 2), (1, 2), (3, 4) AS t(MyCol, UPPER_COL)
-        """
-      Then query result
-        | count(DISTINCT MyCol, UPPER_COL) |
-        | 2                                |
-
-  Rule: COUNT(DISTINCT *) with NULLs
-
-    Scenario: count distinct star skips rows where any column is null
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, 'a'), (NULL, 'b'), (1, 'a') AS t(a, b)
-        """
-      Then query result
-        | count(DISTINCT a, b) |
-        | 1                    |
-
-    Scenario: count distinct star all nulls returns zero
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (NULL, NULL), (NULL, NULL) AS t(a, b)
-        """
-      Then query result
-        | count(DISTINCT a, b) |
-        | 0                    |
-
-    Scenario: count distinct star with some null columns
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, NULL), (2, NULL), (1, NULL) AS t(a, b)
-        """
-      Then query result
-        | count(DISTINCT a, b) |
-        | 0                    |
-
-    Scenario: count distinct star mixed nulls and values
-      When query
-        """
-        SELECT COUNT(DISTINCT *) FROM VALUES (1, 'a'), (1, NULL), (2, 'b'), (2, 'b'), (NULL, NULL) AS t(a, b)
-        """
-      Then query result
-        | count(DISTINCT a, b) |
-        | 2                    |
diff --git a/python/pysail/tests/spark/function/features/count_if.feature b/python/pysail/tests/spark/function/features/count_if.feature
deleted file mode 100644
index c78afa11d..000000000
--- a/python/pysail/tests/spark/function/features/count_if.feature
+++ /dev/null
@@ -1,19 +0,0 @@
-Feature: count_if function
-
-  Rule: count_if as window function
-
-    Scenario: count_if over window
-      When query
-      """
-      SELECT value,
-             count_if(value > 2) OVER (ORDER BY value ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS result
-      FROM VALUES (1), (2), (3), (4), (5) AS t(value)
-      ORDER BY value
-      """
-      Then query result ordered
-      | value | result |
-      | 1     | 0      |
-      | 2     | 0      |
-      | 3     | 1      |
-      | 4     | 2      |
-      | 5     | 3      |
diff --git a/python/pysail/tests/spark/function/features/csv/csv_options.feature b/python/pysail/tests/spark/function/features/csv/csv_options.feature
new file mode 100644
index 000000000..698186784
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/csv/csv_options.feature
@@ -0,0 +1,1014 @@
+Feature: CSV expression functions handle Spark's CSV options
+
+  Spark builds `CSVOptions` eagerly, so every option is parsed as soon as the options map is read,
+  even by a function that never uses the resulting value. A bad option is rejected by `from_csv`,
+  `to_csv`, and `schema_of_csv` alike, including for options that function ignores.
+
+  Rule: A non-boolean value is rejected by from_csv
+
+    Scenario Outline: from_csv rejects a non-boolean 
    10.4
    ', 'a INT, b DOUBLE') AS r
+        """
+      Then query result
+        | r        |
+        | {1, 0.4} |
+
+    Scenario Outline: Primitive: 
+      When query
+        """
+        SELECT from_xml(, ). AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                | xml                         | schema         | field | result |
+        | Parse STRING field  | '
    sunset
    '      | 's STRING'     | s     | sunset |
+        | Parse BOOLEAN true  | '
    true
    '  | 'flag BOOLEAN' | flag  | true   |
+        | Parse BOOLEAN false | '
    false
    ' | 'flag BOOLEAN' | flag  | false  |
+        | Parse LONG field    | '
    43
    '          | 'x LONG'       | x     | 43     |
+        | Parse FLOAT field   | '
    3.5
    '         | 'x FLOAT'      | x     | 3.5    |
+
+  Rule: Root tag is ignored
+
+    Scenario: Root tag name does not affect parsing
+      When query
+        """
+        SELECT
+          from_xml('4',      'a INT').a AS r1,
+          from_xml('4', 'a INT').a AS r2,
+          from_xml('
    4
    ',           'a INT').a AS r3
+        """
+      Then query result
+        | r1 | r2 | r3 |
+        | 4  | 4  | 4  |
+
+  Rule: NULL handling
+
+    Scenario: NULL input string returns NULL struct
+      When query
+        """
+        SELECT from_xml(CAST(NULL AS STRING), 'a INT') AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: Missing tag returns NULL field
+      When query
+        """
+        SELECT from_xml('
    1
    ', 'a INT, b INT') AS r
+        """
+      Then query result
+        | r         |
+        | {NULL, 1} |
+
+    Scenario: Empty tag with INT schema returns NULL
+      When query
+        """
+        SELECT from_xml('
    ', 'a INT').a AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: Empty tag with STRING schema returns empty string
+      When query
+        """
+        SELECT from_xml('
    ', 'a STRING').a AS result
+        """
+      Then query result
+        | result |
+        |        |
+
+    Scenario: Multi-row with missing field and NULL row
+      When query
+        """
+        SELECT from_xml(xml_col, 'a INT, b STRING') AS r
+        FROM VALUES
+          ('
    1x
    '),
+          ('
    2y
    '),
+          ('
    3
    '),
+          (CAST(NULL AS STRING))
+        AS t(xml_col)
+        """
+      Then query result
+        | r         |
+        | {1, x}    |
+        | {2, y}    |
+        | {3, NULL} |
+        | NULL      |
+
+  Rule: nullValue option
+
+    Scenario: nullValue option maps matching string to NULL for INT
+      When query
+        """
+        SELECT from_xml('
    N/A
    ', 'a INT', map('nullValue', 'N/A')).a AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: nullValue option maps matching string to NULL for STRING
+      When query
+        """
+        SELECT from_xml('
    N/A
    ', 'a STRING', map('nullValue', 'N/A')).a AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: nullValue empty string turns empty tag into NULL for STRING
+      When query
+        """
+        SELECT from_xml('
    ', 'a STRING', map('nullValue', '')).a AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  Rule: Nested structures
+
+    Scenario: Nested struct becomes nested field access
+      When query
+        """
+        SELECT from_xml(
+          '
    PabloAna1
    ',
+          'teacher STRING, student STRUCT'
+        ) AS r
+        """
+      Then query result
+        | r                 |
+        | {Pablo, {Ana, 1}} |
+
+    Scenario: Three levels of nesting
+      When query
+        """
+        SELECT from_xml('
    45
    ', 'a STRUCT>').a.b.c AS result
+        """
+      Then query result
+        | result |
+        | 45     |
+
+  Rule: Arrays
+
+    Scenario: Repeated tags produce array of primitives
+      When query
+        """
+        SELECT from_xml('
    123
    ', 'a ARRAY').a AS result
+        """
+      Then query result
+        | result    |
+        | [1, 2, 3] |
+
+    Scenario: Repeated struct tags produce array of structs
+      When query
+        """
+        SELECT from_xml(
+          '
    Bob1Charlie2
    ',
+          'student ARRAY>'
+        ).student AS result
+        """
+      Then query result
+        | result                   |
+        | [{Bob, 1}, {Charlie, 2}] |
+
+  Rule: Date and timestamp types
+
+    Scenario Outline: Default format: 
+      When query
+        """
+        SELECT from_xml(, ). AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                                    | xml                                       | schema             | field | result              |
+        | Parse DATE with default format          | '
    2026-06-17
    '                | 'd DATE'           | d     | 2026-06-17          |
+        | Parse TIMESTAMP with default format     | '
    2026-06-17T18:00:00.000
    ' | 'ts TIMESTAMP'     | ts    | 2026-06-17 18:00:00 |
+        | Parse TIMESTAMP_NTZ with default format | '
    2026-06-17T18:00:00.000
    ' | 'ts TIMESTAMP_NTZ' | ts    | 2026-06-17 18:00:00 |
+
+    Scenario: Custom timestampFormat option
+      When query
+        """
+        SELECT from_xml(
+          '
    17/06/2026
    ',
+          'ts TIMESTAMP',
+          map('timestampFormat', 'dd/MM/yyyy')
+        ).ts AS result
+        """
+      Then query result
+        | result              |
+        | 2026-06-17 00:00:00 |
+
+    Scenario: Custom timestampNTZFormat option
+      When query
+        """
+        SELECT from_xml(
+          '
    17/06/2026 18:00
    ',
+          'ts TIMESTAMP_NTZ',
+          map('timestampNTZFormat', 'dd/MM/yyyy HH:mm')
+        ).ts AS result
+        """
+      Then query result
+        | result              |
+        | 2026-06-17 18:00:00 |
+
+    Scenario: From XML Java datetime pattern contract for quoted literals and fractions
+      When query
+        """
+        SELECT
+          CAST(from_xml(
+            '
    2026B06B17
    ',
+            'd DATE',
+            map(
+              'dateFormat',
+              concat(
+                'yyyy', chr(39), 'B', chr(39),
+                'MM', chr(39), 'B', chr(39), 'dd'
+              )
+            )
+          ).d AS STRING) AS date_result,
+          CAST(from_xml(
+            '
    2026-06-17 18:00:01.1
    ',
+            'ts TIMESTAMP',
+            map('timestampFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS')
+          ).ts AS STRING) AS timestamp_result
+        """
+      Then query result
+        | date_result | timestamp_result      |
+        | 2026-06-17  | 2026-06-17 18:00:01.1 |
+
+    Scenario: From XML Java datetime pattern contract for timestamp zones
+      Given config spark.sql.session.timeZone = America/Los_Angeles
+      When query
+        """
+        SELECT
+          CAST(from_xml(
+            '
    2026-06-17T18:00:00+02:00
    ',
+            'ts TIMESTAMP',
+            map(
+              'timestampFormat',
+              concat('yyyy-MM-dd', chr(39), 'T', chr(39), 'HH:mm:ssXXX')
+            )
+          ).ts AS STRING) AS timestamp_ltz,
+          CAST(from_xml(
+            '
    2026-06-17T18:00:00
    ',
+            'ts TIMESTAMP_NTZ',
+            map(
+              'timestampNTZFormat',
+              concat('yyyy-MM-dd', chr(39), 'T', chr(39), 'HH:mm:ss')
+            )
+          ).ts AS STRING) AS timestamp_ntz
+        """
+      Then query result
+        | timestamp_ltz       | timestamp_ntz       |
+        | 2026-06-17 09:00:00 | 2026-06-17 18:00:00 |
+
+  Rule: XML entity unescaping
+
+    Scenario Outline: Entity: 
+      When query
+        """
+        SELECT from_xml(, 's STRING').s AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                             | xml                                | result   |
+        | Less-than entity is unescaped    | '
    a < b
    '           | a < b    |
+        | Ampersand entity is unescaped    | '
    a & b
    '          | a & b    |
+        | Greater-than entity is unescaped | '
    a > b
    '           | a > b    |
+        | Quote entity is unescaped        | '
    say "hi"
    ' | say "hi" |
+
+  Rule: Attributes
+
+    Scenario: Attributes are parsed with default underscore prefix
+      When query
+        """
+        SELECT from_xml('
    Agua
    ', '_id STRING, name STRING') AS r
+        """
+      Then query result
+        | r          |
+        | {99, Agua} |
+
+    Scenario: Custom attributePrefix option
+      When query
+        """
+        SELECT from_xml('
    Amber
    ', 'id STRING, name STRING', map('attributePrefix', '')) AS r
+        """
+      Then query result
+        | r           |
+        | {99, Amber} |
+
+  Rule: valueTag option
+
+    Scenario: Text content of element is captured via default valueTag field
+      When query
+        """
+        SELECT from_xml('
    colors
    ', '_id STRING, _VALUE STRING') AS r
+        """
+      Then query result
+        | r           |
+        | {7, colors} |
+
+  Rule: Parse mode
+
+    Scenario: PERMISSIVE mode returns NULL field for bad cast
+      When query
+        """
+        SELECT from_xml('
    not_a_number
    ', 'a INT').a AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: Explicit PERMISSIVE mode returns NULL field for bad cast
+      When query
+        """
+        SELECT from_xml('
    not_a_number
    ', 'a INT', map('mode', 'PERMISSIVE')).a AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+  Rule: Decimal type
+
+    Scenario Outline: Decimal: 
+      When query
+        """
+        SELECT from_xml(, ).a AS result
+        """
+      Then query result
+        | result   |
+        |  |
+
+      Examples:
+        | case                           | xml                             | schema                   | result       |
+        | Parse DECIMAL field            | '
    2.12
    '            | 'a DECIMAL(10,2)'        | 2.12         |
+        | DECIMAL truncates to scale     | '
    2.12159
    '         | 'a DECIMAL(10,2)'        | 2.12         |
+        | DECIMAL overflow returns NULL  | '
    99999999999.99
    '  | 'a DECIMAL(10,2)'        | NULL         |
+        | DECIMAL bad value returns NULL | '
    bad
    '             | 'a DECIMAL(10,2)'        | NULL         |
+        | DECIMAL in array               | '
    1.102.20
    ' | 'a ARRAY' | [1.10, 2.20] |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null xml literal yields a struct
+      When query
+        """
+        SELECT from_xml('1', 'a INT') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: struct (nullable = true)
+         |    |-- a: integer (nullable = true)
+        """
+
+    Scenario: a non-null xml column yields a struct
+      When query
+        """
+        SELECT from_xml(CONCAT('', CAST(id AS STRING), ''), 'a INT') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: struct (nullable = true)
+         |    |-- a: integer (nullable = true)
+        """
+
+    Scenario: a nullable xml column stays nullable
+      When query
+        """
+        SELECT from_xml(c, 'a INT') AS result FROM VALUES ('1'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: struct (nullable = true)
+         |    |-- a: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/to_xml.feature b/python/pysail/tests/spark/function/features/xml/to_xml.feature
similarity index 50%
rename from python/pysail/tests/spark/function/features/to_xml.feature
rename to python/pysail/tests/spark/function/features/xml/to_xml.feature
index 712511c7e..5a9c19886 100644
--- a/python/pysail/tests/spark/function/features/to_xml.feature
+++ b/python/pysail/tests/spark/function/features/xml/to_xml.feature
@@ -40,15 +40,6 @@ Feature: to_xml converts a struct value to an XML string
         | 2 | 2 |
         | 3 | 3 |
 
-    Scenario: XML declaration is present by default
-      When query
-        """
-        SELECT to_xml(named_struct('a', 1)) LIKE '
       When query
         """
-        SELECT xpath_string(to_xml(named_struct('msg', 'a > b')), '/ROW/msg') AS result
+        SELECT xpath_string(to_xml(named_struct('msg', )), '/ROW/msg') AS result
         """
       Then query result
-        | result |
-        | a > b  |
-
-    Scenario: Ampersand character is escaped
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('msg', 'a & b')), '/ROW/msg') AS result
-        """
-      Then query result
-        | result |
-        | a & b  |
-
-    Scenario: Multiple special characters together
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('msg', 'a < b & c > d')), '/ROW/msg') AS result
-        """
-      Then query result
-        | result        |
-        | a < b & c > d |
+        | result   |
+        |  |
 
-    Scenario: Script tag content is safely escaped
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('msg', '')), '/ROW/msg') AS result
-        """
-      Then query result
-        | result                       |
-        |  |
+      Examples:
+        | case                                 | value                           | result                        |
+        | Less-than character is escaped       | 'a < b'                         | a < b                         |
+        | Greater-than character is escaped    | 'a > b'                         | a > b                         |
+        | Ampersand character is escaped       | 'a & b'                         | a & b                         |
+        | Multiple special characters together | 'a < b & c > d'                 | a < b & c > d                 |
+        | Script tag content is safely escaped | '' |  |
 
   Rule: Timestamp and date formatting
 
@@ -353,68 +280,24 @@ Feature: to_xml converts a struct value to an XML string
 
   Rule: Decimal and special values
 
-    Scenario: Decimal field is formatted as fixed-point string
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('price', CAST(2.99 AS DECIMAL(5,2)))), '/ROW/price') AS result
-        """
-      Then query result
-        | result |
-        | 2.99   |
-
-    Scenario: Negative decimal preserves sign
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('price', CAST(-0.83 AS DECIMAL(5,2)))), '/ROW/price') AS result
-        """
-      Then query result
-        | result |
-        | -0.83  |
-
-    Scenario: Very large decimal value
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('big', CAST(999999.99 AS DECIMAL(10,2)))), '/ROW/big') AS result
-        """
-      Then query result
-        | result    |
-        | 999999.99 |
-
-    Scenario: Very small decimal value
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('tiny', CAST(0.0001 AS DECIMAL(5,4)))), '/ROW/tiny') AS result
-        """
-      Then query result
-        | result |
-        | 0.0001 |
-
-    Scenario: Floating-point NaN value
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('nan', CAST('NaN' AS DOUBLE))), '/ROW/nan') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
-
-    Scenario: Floating-point positive infinity
+    Scenario Outline: Decimal and special values: 
       When query
         """
-        SELECT xpath_string(to_xml(named_struct('pos', CAST('Infinity' AS DOUBLE))), '/ROW/pos') AS result
+        SELECT xpath_string(to_xml(named_struct(, )), ) AS result
         """
       Then query result
         | result   |
-        | Infinity |
-
-    Scenario: Floating-point negative infinity
-      When query
-        """
-        SELECT xpath_string(to_xml(named_struct('neg', CAST('-Infinity' AS DOUBLE))), '/ROW/neg') AS result
-        """
-      Then query result
-        | result    |
-        | -Infinity |
+        |  |
+
+      Examples:
+        | case                                             | field   | value                            | path         | result    |
+        | Decimal field is formatted as fixed-point string | 'price' | CAST(2.99 AS DECIMAL(5,2))       | '/ROW/price' | 2.99      |
+        | Negative decimal preserves sign                  | 'price' | CAST(-0.83 AS DECIMAL(5,2))      | '/ROW/price' | -0.83     |
+        | Very large decimal value                         | 'big'   | CAST(999999.99 AS DECIMAL(10,2)) | '/ROW/big'   | 999999.99 |
+        | Very small decimal value                         | 'tiny'  | CAST(0.0001 AS DECIMAL(5,4))     | '/ROW/tiny'  | 0.0001    |
+        | Floating-point NaN value                         | 'nan'   | CAST('NaN' AS DOUBLE)            | '/ROW/nan'   | NaN       |
+        | Floating-point positive infinity                 | 'pos'   | CAST('Infinity' AS DOUBLE)       | '/ROW/pos'   | Infinity  |
+        | Floating-point negative infinity                 | 'neg'   | CAST('-Infinity' AS DOUBLE)      | '/ROW/neg'   | -Infinity |
 
   Rule: Complex structures
 
@@ -449,27 +332,24 @@ Feature: to_xml converts a struct value to an XML string
         | result |
         | false  |
 
-    Scenario: Struct with array and nested struct
-      When query
-        """
-        SELECT
-          xpath(to_xml(named_struct('items', array(1, 2), 'nested', named_struct('value', 42))), '/ROW/items/item/text()') AS items,
-          xpath_string(to_xml(named_struct('items', array(1, 2), 'nested', named_struct('value', 42))), '/ROW/nested/value') AS nested
-        """
-      Then query result
-        | items  | nested |
-        | [1, 2] | 42     |
-
   Rule: Options
 
-    Scenario: Custom rowTag changes root element name
+    Scenario Outline: Option: 
       When query
         """
-        SELECT to_xml(named_struct('a', 1), map('rowTag', 'Person')) LIKE '%%' AS result
+        SELECT to_xml() LIKE  AS result
         """
       Then query result
-        | result |
-        | true   |
+        | result   |
+        |  |
+
+      Examples:
+        | case                                                      | args                                                          | pattern        | result |
+        | Custom rowTag changes root element name                   | named_struct('a', 1), map('rowTag', 'Person')                 | '%%'   | true   |
+        | Option keys are case-insensitive                          | named_struct('a', 1), map('ROWTAG', 'Item')                   | '%%'     | true   |
+        | Suppressed declaration when empty string passed           | named_struct('a', 1), map('declaration', '')                  | '%N/A%' | true   |
+        | Attribute prefix fields become XML attributes             | named_struct('_id', 99, 'name', 'Alice')                      | '%id="99"%'    | true   |
 
     Scenario: valueTag field is written as inline text content on the parent element
       When query
@@ -491,51 +371,6 @@ Feature: to_xml converts a struct value to an XML string
         | has_content | has_element |
         | true        | false       |
 
-    Scenario: Option keys are case-insensitive
-      When query
-        """
-        SELECT to_xml(named_struct('a', 1), map('ROWTAG', 'Item')) LIKE '%%' AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: Custom arrayElementName changes inner element tag
-      When query
-        """
-        SELECT to_xml(named_struct('nums', array(10, 20)), map('arrayElementName', 'val')) LIKE '%10%' AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: Suppressed declaration when empty string passed
-      When query
-        """
-        SELECT to_xml(named_struct('a', 1), map('declaration', '')) LIKE '%N/A%' AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
-    Scenario: Attribute prefix fields become XML attributes
-      When query
-        """
-        SELECT to_xml(named_struct('_id', 99, 'name', 'Alice')) LIKE '%id="99"%' AS result
-        """
-      Then query result
-        | result |
-        | true   |
-
   Rule: Map fields
 
     Scenario: Map keys become child tag names
@@ -613,3 +448,301 @@ Feature: to_xml converts a struct value to an XML string
       Then query result
         | result |
         | [1, 2] |
+
+  # ============================================================
+  # Bug-hunt (2026-07-10): JVM-validated exact-output scenarios.
+  # Flatten with replace(..., '\n', '~'); '~' marks a newline so the
+  # exact bytes (declaration, indentation, tag structure) are asserted.
+  # ============================================================
+
+  Rule: Serialization envelope
+
+    Scenario: No XML declaration, no trailing newline, four-space indent
+      When query
+        """
+        SELECT replace(to_xml(named_struct('a', 1, 'b', 2)), '\n', '~') AS r
+        """
+      Then query result
+        | r                                      |
+        | ~    1~    2~ |
+
+    Scenario: Struct whose only field is NULL is self-closing
+      When query
+        """
+        SELECT replace(to_xml(named_struct('a', CAST(NULL AS INT))), '\n', '~') AS r
+        """
+      Then query result
+        | r      |
+        |  |
+
+    Scenario: declaration option is ignored for string output
+      When query
+        """
+        SELECT replace(to_xml(named_struct('a', 1), map('declaration', 'version="1.0"')), '\n', '~') AS r
+        """
+      Then query result
+        | r                         |
+        | ~    1~ |
+
+  Rule: Primitive arrays are repeated flat tags
+
+    Scenario Outline: Primitive array: 
+      When query
+        """
+        SELECT replace(to_xml(named_struct()), '\n', '~') AS r
+        """
+      Then query result
+        | r        |
+        |  |
+
+      Examples:
+        | case                                                             | args                                                       | result                                                                                                              |
+        | Primitive array repeats the field tag without an item wrapper    | 'numbers', array(1, 2, 3)                                  | ~    1~    2~    3~                             |
+        | Single-element primitive array                                   | 'items', array(1)                                          | ~    1~                                                                                   |
+        | NULL elements are dropped from a primitive array                 | 'items', array(1, CAST(NULL AS INT), 3)                    | ~    1~    3~                                                              |
+        | Nested arrays wrap inner elements in item, outer keeps field tag | 'm', array(array(1, 2), array(3))                          | ~    ~        1~        2~    ~    ~        3~    ~ |
+        | Array of structs repeats the field tag around struct fields      | 'items', array(named_struct('x', 1), named_struct('x', 2)) | ~    ~        1~    ~    ~        2~    ~                    |
+
+  Rule: Binary fields
+
+    Scenario: Binary renders as bracketed uppercase hex
+      When query
+        """
+        SELECT replace(to_xml(named_struct('b', X'48656C6C6F')), '\n', '~') AS r
+        """
+      Then query result
+        | r                                        |
+        | ~    [48 65 6C 6C 6F]~ |
+
+    Scenario: Empty binary renders as empty brackets
+      When query
+        """
+        SELECT replace(to_xml(named_struct('b', X'')), '\n', '~') AS r
+        """
+      Then query result
+        | r                          |
+        | ~    []~ |
+
+  Rule: XML escaping
+
+    Scenario: Text escapes ampersand and less-than but leaves greater-than literal
+      When query
+        """
+        SELECT replace(to_xml(named_struct('msg', 'a < b > c & d')), '\n', '~') AS r
+        """
+      Then query result
+        | r                                                |
+        | ~    a < b > c & d~ |
+
+    Scenario: Attribute escapes ampersand, less-than and double-quote but not greater-than
+      When query
+        """
+        SELECT replace(to_xml(named_struct('_a', 'xz&"q')), '\n', '~') AS r
+        """
+      Then query result
+        | r                               |
+        |  |
+
+  Rule: valueTag is inline text content
+
+    Scenario: valueTag-only struct glues text with no surrounding whitespace
+      When query
+        """
+        SELECT replace(to_xml(named_struct('_VALUE', 'hello')), '\n', '~') AS r
+        """
+      Then query result
+        | r                |
+        | hello |
+
+    Scenario: valueTag with an attribute sibling
+      When query
+        """
+        SELECT replace(to_xml(named_struct('_id', 1, '_VALUE', 'hello')), '\n', '~') AS r
+        """
+      Then query result
+        | r                       |
+        | hello |
+
+    Scenario: valueTag text precedes element siblings inline
+      When query
+        """
+        SELECT replace(to_xml(named_struct('_VALUE', 'hi', 'child', 5)), '\n', '~') AS r
+        """
+      Then query result
+        | r                                   |
+        | hi~    5~ |
+
+  Rule: Maps
+
+    Scenario: Map keys become child tags
+      When query
+        """
+        SELECT replace(to_xml(named_struct('m', map('k1', 1, 'k2', 2))), '\n', '~') AS r
+        """
+      Then query result
+        | r                                                                   |
+        | ~    ~        1~        2~    ~ |
+
+    Scenario: Map with array value repeats the key tag
+      When query
+        """
+        SELECT replace(to_xml(named_struct('m', map('nums', array(1, 2, 3)))), '\n', '~') AS r
+        """
+      Then query result
+        | r                                                                                                  |
+        | ~    ~        1~        2~        3~    ~ |
+
+  Rule: NULL handling and options
+
+    Scenario: NULL struct input returns NULL exact
+      When query
+        """
+        SELECT to_xml(CAST(NULL AS STRUCT)) AS r
+        """
+      Then query result
+        | r    |
+        | NULL |
+
+    Scenario: NULL field is omitted by default
+      When query
+        """
+        SELECT replace(to_xml(named_struct('a', 1, 'b', CAST(NULL AS INT))), '\n', '~') AS r
+        """
+      Then query result
+        | r                         |
+        | ~    1~ |
+
+    Scenario: Empty string field is an open-close pair, not self-closing
+      When query
+        """
+        SELECT replace(to_xml(named_struct('text', '')), '\n', '~') AS r
+        """
+      Then query result
+        | r                              |
+        | ~    ~ |
+
+    Scenario: nullValue option renders NULL fields with the configured string
+      When query
+        """
+        SELECT replace(to_xml(named_struct('a', CAST(NULL AS INT)), map('nullValue', 'NA')), '\n', '~') AS r
+        """
+      Then query result
+        | r                          |
+        | ~    NA~ |
+
+  Rule: Temporal and decimal formatting
+
+    Scenario: Date uses default yyyy-MM-dd format
+      When query
+        """
+        SELECT replace(to_xml(named_struct('d', DATE '2026-06-06')), '\n', '~') AS r
+        """
+      Then query result
+        | r                                  |
+        | ~    2026-06-06~ |
+
+    Scenario: Decimal keeps trailing zeros to scale
+      When query
+        """
+        SELECT replace(to_xml(named_struct('price', CAST(2.99 AS DECIMAL(5,2)))), '\n', '~') AS r
+        """
+      Then query result
+        | r                                    |
+        | ~    2.99~ |
+
+    Scenario: TIMESTAMP_NTZ uses ISO 8601 with millisecond precision and no offset
+      When query
+        """
+        SELECT replace(to_xml(named_struct('ts', CAST('2026-06-06 12:00:00' AS TIMESTAMP_NTZ))), '\n', '~') AS r
+        """
+      Then query result
+        | r                                                 |
+        | ~    2026-06-06T12:00:00.000~ |
+
+  Rule: Special float values
+
+    Scenario: NaN renders literally
+      When query
+        """
+        SELECT replace(to_xml(named_struct('d', CAST('NaN' AS DOUBLE))), '\n', '~') AS r
+        """
+      Then query result
+        | r                           |
+        | ~    NaN~ |
+
+    Scenario: Positive infinity renders as Infinity
+      When query
+        """
+        SELECT replace(to_xml(named_struct('d', CAST('Infinity' AS DOUBLE))), '\n', '~') AS r
+        """
+      Then query result
+        | r                                |
+        | ~    Infinity~ |
+
+  Rule: Double and float use Java toString formatting
+
+    Scenario Outline: Java toString formatting: 
+      When query
+        """
+        SELECT replace(to_xml(named_struct()), '\n', '~') AS r
+        """
+      Then query result
+        | r        |
+        |  |
+
+      Examples:
+        | case                                                     | args                                                 | result                                     |
+        | Whole double keeps a decimal point                       | 'd', CAST(1.0 AS DOUBLE)                             | ~    1.0~                |
+        | Large double uses scientific notation                    | 'd', CAST(1e20 AS DOUBLE)                            | ~    1.0E20~             |
+        | Small double uses scientific notation                    | 'd', CAST(1e-7 AS DOUBLE)                            | ~    1.0E-7~             |
+        | Medium whole double keeps a decimal point                | 'd', CAST(123456.0 AS DOUBLE)                        | ~    123456.0~           |
+        | Negative zero double normalizes to positive zero         | 'd', CAST(-0.0 AS DOUBLE)                            | ~    0.0~                |
+        | Whole float keeps a decimal point                        | 'f', CAST(1.0 AS FLOAT)                              | ~    1.0~                |
+        | Double array repeats the field tag with formatted values | 'v', array(CAST(1.0 AS DOUBLE), CAST(2.5 AS DOUBLE)) | ~    1.0~    2.5~ |
+
+  Rule: Array element naming option
+
+    Scenario: Custom arrayElementName names the inner elements of nested arrays
+      When query
+        """
+        SELECT replace(to_xml(named_struct('m', array(array(10, 20))), map('arrayElementName', 'val')), '\n', '~') AS r
+        """
+      Then query result
+        | r                                                                         |
+        | ~    ~        10~        20~    ~ |
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null struct literal is nullable (to_xml is inherently nullable in Spark)
+      When query
+        """
+        SELECT to_xml(struct(1, 'a')) AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
+
+    Scenario: a non-null struct column is nullable (to_xml is inherently nullable in Spark)
+      When query
+        """
+        SELECT to_xml(struct(id, id)) AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
+
+    Scenario: a nullable struct column stays nullable
+      When query
+        """
+        SELECT to_xml(c) AS result FROM VALUES (named_struct('a', 1)), (CAST(NULL AS STRUCT)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath.feature b/python/pysail/tests/spark/function/features/xml/xpath.feature
new file mode 100644
index 000000000..1327a1603
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath.feature
@@ -0,0 +1,148 @@
+Feature: xpath() extracts XML nodes with Spark-compatible semantics
+
+  Rule: Node selection returns arrays of string-like values
+
+    Scenario: xpath returns text nodes in document order
+      When query
+        """
+        SELECT xpath('b1b2b3', 'a/b/text()') AS result
+        """
+      Then query result
+        | result       |
+        | [b1, b2, b3] |
+
+    Scenario: xpath returns NULL entries for element nodes
+      When query
+        """
+        SELECT xpath('b1b2', 'a/b') AS result
+        """
+      Then query result
+        | result       |
+        | [NULL, NULL] |
+
+    Scenario: xpath returns an empty list when no nodes match
+      When query
+        """
+        SELECT xpath('1', 'a/c') AS result
+        """
+      Then query result
+        | result |
+        | []     |
+
+  Rule: Empty or null inputs return NULL
+
+    Scenario: xpath returns NULL for empty xml
+      When query
+        """
+        SELECT xpath('', 'a/b') AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: xpath returns NULL for empty path
+      When query
+        """
+        SELECT xpath('1', '') AS result
+        """
+      Then query result
+        | result |
+        | NULL   |
+
+    Scenario: xpath returns NULL for null xml or path
+      When query
+        """
+        SELECT
+          xpath(CAST(NULL AS STRING), 'a/b') AS null_xml,
+          xpath('1', CAST(NULL AS STRING)) AS null_path
+        """
+      Then query result
+        | null_xml | null_path |
+        | NULL     | NULL      |
+
+  Rule: Non-node XPath results fail
+
+    Scenario: xpath rejects expressions that do not return a node list
+      When query
+        """
+        SELECT xpath('1', 'sum(a/b)') AS result
+        """
+      Then query error (?s).*NodeList.*
+
+  Rule: xpath — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath with the argument as a literal
+      When query
+        """
+        SELECT xpath('b1b2b3c1c2','a/b') AS result
+        """
+      Then query result ordered
+        | result             |
+        | [NULL, NULL, NULL] |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ["['b1', 'b2', 'b3']", '[None, None, None]'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath takes argument 2 from a column holding two different values
+      When query
+        """
+        SELECT xpath('b1b2b3c1c2', c) AS result FROM VALUES (1, 'a/b/text()'), (2, 'a/b') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['[None, None, None]', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath('b1b2b3c1c2', c) AS result FROM VALUES (1, 'a/b'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['[None, None, None]', '[None, None, None]'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath takes argument 2 from a column
+      When query
+        """
+        SELECT xpath('b1b2b3c1c2', c) AS result FROM VALUES (1, 'a/b'), (2, 'a/b') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null xml literal yields an array
+      When query
+        """
+        SELECT xpath('12', 'a/b/text()') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: string (containsNull = true)
+        """
+
+    Scenario: a non-null xml column yields an array
+      When query
+        """
+        SELECT xpath(CONCAT('', CAST(id AS STRING), ''), 'a/b/text()') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: string (containsNull = true)
+        """
+
+    Scenario: a nullable xml column stays nullable
+      When query
+        """
+        SELECT xpath(c, 'a/b/text()') AS result FROM VALUES ('1'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: array (nullable = true)
+         |    |-- element: string (containsNull = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_boolean.feature b/python/pysail/tests/spark/function/features/xml/xpath_boolean.feature
new file mode 100644
index 000000000..aadc40a5b
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_boolean.feature
@@ -0,0 +1,70 @@
+Feature: xpath_boolean with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_boolean — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_boolean with the argument as a literal
+      When query
+        """
+        SELECT xpath_boolean('1','a/b') AS result
+        """
+      Then query result ordered
+        | result |
+        | true   |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['True', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_boolean takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_boolean('1', c) AS result FROM VALUES (1, 'a/b'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['True', 'True'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_boolean takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_boolean('1', c) AS result FROM VALUES (1, 'a/b'), (2, 'a/b') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null xml literal yields a boolean
+      When query
+        """
+        SELECT xpath_boolean('1', 'a/b=1') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
+
+    Scenario: a non-null xml column yields a boolean
+      When query
+        """
+        SELECT xpath_boolean(CONCAT('', CAST(id AS STRING), ''), 'a/b=1') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
+
+    Scenario: a nullable xml column stays nullable
+      When query
+        """
+        SELECT xpath_boolean(c, 'a/b=1') AS result FROM VALUES ('1'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: boolean (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_double.feature b/python/pysail/tests/spark/function/features/xml/xpath_double.feature
new file mode 100644
index 000000000..e968dd3b3
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_double.feature
@@ -0,0 +1,34 @@
+Feature: xpath_double with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_double — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_double with the argument as a literal
+      When query
+        """
+        SELECT xpath_double('12', 'sum(a/b)') AS result
+        """
+      Then query result ordered
+        | result |
+        | 3.0    |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3.0', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_double takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_double('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3.0', '3.0'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_double takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_double('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_float.feature b/python/pysail/tests/spark/function/features/xml/xpath_float.feature
new file mode 100644
index 000000000..9fc28cd9c
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_float.feature
@@ -0,0 +1,70 @@
+Feature: xpath_float with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_float — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_float with the argument as a literal
+      When query
+        """
+        SELECT xpath_float('12', 'sum(a/b)') AS result
+        """
+      Then query result ordered
+        | result |
+        | 3.0    |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3.0', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_float takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_float('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3.0', '3.0'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_float takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_float('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to xpath_float yields the schema Spark declares
+      When query
+        """
+        SELECT xpath_float('12', 'sum(a/b)') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: float (nullable = true)
+        """
+
+    Scenario: a non-null column input to xpath_float yields the schema Spark declares
+      When query
+        """
+        SELECT xpath_float(CAST(id AS STRING), 'sum(a/b)') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: float (nullable = true)
+        """
+
+    Scenario: a nullable column input to xpath_float stays nullable
+      When query
+        """
+        SELECT xpath_float(c, 'sum(a/b)') AS result FROM VALUES ('12'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: float (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_int.feature b/python/pysail/tests/spark/function/features/xml/xpath_int.feature
new file mode 100644
index 000000000..4777c0dc3
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_int.feature
@@ -0,0 +1,70 @@
+Feature: xpath_int with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_int — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_int with the argument as a literal
+      When query
+        """
+        SELECT xpath_int('12', 'sum(a/b)') AS result
+        """
+      Then query result ordered
+        | result |
+        | 3      |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_int takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_int('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3', '3'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_int takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_int('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null xml literal yields an integer
+      When query
+        """
+        SELECT xpath_int('5', 'a/b') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a non-null xml column yields an integer
+      When query
+        """
+        SELECT xpath_int(CONCAT('', CAST(id AS STRING), ''), 'a/b') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
+
+    Scenario: a nullable xml column stays nullable
+      When query
+        """
+        SELECT xpath_int(c, 'a/b') AS result FROM VALUES ('5'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: integer (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_long.feature b/python/pysail/tests/spark/function/features/xml/xpath_long.feature
new file mode 100644
index 000000000..7a3d7e877
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_long.feature
@@ -0,0 +1,70 @@
+Feature: xpath_long with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_long — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_long with the argument as a literal
+      When query
+        """
+        SELECT xpath_long('12', 'sum(a/b)') AS result
+        """
+      Then query result ordered
+        | result |
+        | 3      |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_long takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_long('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3', '3'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_long takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_long('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to xpath_long yields the schema Spark declares
+      When query
+        """
+        SELECT xpath_long('12', 'sum(a/b)') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: a non-null column input to xpath_long yields the schema Spark declares
+      When query
+        """
+        SELECT xpath_long(CAST(id AS STRING), 'sum(a/b)') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
+
+    Scenario: a nullable column input to xpath_long stays nullable
+      When query
+        """
+        SELECT xpath_long(c, 'sum(a/b)') AS result FROM VALUES ('12'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: long (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_number.feature b/python/pysail/tests/spark/function/features/xml/xpath_number.feature
new file mode 100644
index 000000000..35d435fe7
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_number.feature
@@ -0,0 +1,34 @@
+Feature: xpath_number with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_number — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_number with the argument as a literal
+      When query
+        """
+        SELECT xpath_number('12', 'sum(a/b)') AS result
+        """
+      Then query result ordered
+        | result |
+        | 3.0    |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3.0', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_number takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_number('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3.0', '3.0'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_number takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_number('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_short.feature b/python/pysail/tests/spark/function/features/xml/xpath_short.feature
new file mode 100644
index 000000000..4437f8903
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_short.feature
@@ -0,0 +1,70 @@
+Feature: xpath_short with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_short — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_short with the argument as a literal
+      When query
+        """
+        SELECT xpath_short('12', 'sum(a/b)') AS result
+        """
+      Then query result ordered
+        | result |
+        | 3      |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_short takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_short('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['3', '3'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_short takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_short('12', c) AS result FROM VALUES (1, 'sum(a/b)'), (2, 'sum(a/b)') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null literal input to xpath_short yields the schema Spark declares
+      When query
+        """
+        SELECT xpath_short('12', 'sum(a/b)') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: short (nullable = true)
+        """
+
+    Scenario: a non-null column input to xpath_short yields the schema Spark declares
+      When query
+        """
+        SELECT xpath_short(CAST(id AS STRING), 'sum(a/b)') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: short (nullable = true)
+        """
+
+    Scenario: a nullable column input to xpath_short stays nullable
+      When query
+        """
+        SELECT xpath_short(c, 'sum(a/b)') AS result FROM VALUES ('12'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: short (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xml/xpath_string.feature b/python/pysail/tests/spark/function/features/xml/xpath_string.feature
new file mode 100644
index 000000000..3d1c23da8
--- /dev/null
+++ b/python/pysail/tests/spark/function/features/xml/xpath_string.feature
@@ -0,0 +1,70 @@
+Feature: xpath_string with an argument coming from a column
+  # A behaviour-governing argument given as a literal is constant-folded, so the literal
+  # scenarios never exercise the columnar kernel. These scenarios pass the same argument
+  # through a column. All expected values were captured on Spark JVM 4.x.
+
+  Rule: xpath_string — the argument must be foldable
+
+    @function(columnargs)
+    Scenario: xpath_string with the argument as a literal
+      When query
+        """
+        SELECT xpath_string('bcc','a/c') AS result
+        """
+      Then query result ordered
+        | result |
+        | cc     |
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['cc', 'NULL'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_string takes argument 2 from a column containing NULL
+      When query
+        """
+        SELECT xpath_string('bcc', c) AS result FROM VALUES (1, 'a/c'), (2, NULL) AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+    # Spark requires a foldable argument here; Sail accepts a column: Sail returns ['cc', 'cc'].
+    @function(columnargs) @sail-bug
+    Scenario: xpath_string takes argument 2 from a column
+      When query
+        """
+        SELECT xpath_string('bcc', c) AS result FROM VALUES (1, 'a/c'), (2, 'a/c') AS t(i, c) ORDER BY i
+        """
+      Then query error NON_FOLDABLE_INPUT
+
+  @function(nullability)
+  Rule: Output schema
+
+    Scenario: a non-null xml literal yields a string
+      When query
+        """
+        SELECT xpath_string('x', 'a/b') AS result
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
+
+    Scenario: a non-null xml column yields a string
+      When query
+        """
+        SELECT xpath_string(CONCAT('', CAST(id AS STRING), ''), 'a/b') AS result FROM range(3)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
+
+    Scenario: a nullable xml column stays nullable
+      When query
+        """
+        SELECT xpath_string(c, 'a/b') AS result FROM VALUES ('x'), (CAST(NULL AS STRING)) AS t(c)
+        """
+      Then query schema
+        """
+        root
+         |-- result: string (nullable = true)
+        """
diff --git a/python/pysail/tests/spark/function/features/xpath_typed.feature b/python/pysail/tests/spark/function/features/xml/xpath_typed.feature
similarity index 52%
rename from python/pysail/tests/spark/function/features/xpath_typed.feature
rename to python/pysail/tests/spark/function/features/xml/xpath_typed.feature
index 3c0a44e74..abf3dca97 100644
--- a/python/pysail/tests/spark/function/features/xpath_typed.feature
+++ b/python/pysail/tests/spark/function/features/xml/xpath_typed.feature
@@ -2,132 +2,100 @@ Feature: xpath_boolean/double/float/int/long/number/short/string extract typed v
 
   Rule: xpath_boolean evaluates XPath to a boolean
 
-    Scenario: xpath_boolean returns true when node exists
+    Scenario Outline: Boolean: 
       When query
         """
-        SELECT xpath_boolean('1', 'a/b') AS result
+        SELECT xpath_boolean('1', '') AS result
         """
       Then query result
-        | result |
-        | true   |
+        | result   |
+        |  |
 
-    Scenario: xpath_boolean returns false when node does not exist
-      When query
-        """
-        SELECT xpath_boolean('1', 'a/c') AS result
-        """
-      Then query result
-        | result |
-        | false  |
+      Examples:
+        | case                                                 | path | result |
+        | xpath_boolean returns true when node exists          | a/b  | true   |
+        | xpath_boolean returns false when node does not exist | a/c  | false  |
 
   Rule: xpath_double and xpath_number evaluate XPath to a double
 
-    Scenario: xpath_double returns a sum as double
+    Scenario Outline: Double: 
       When query
         """
-        SELECT xpath_double('12', 'sum(a/b)') AS result
+        SELECT ('', '') AS result
         """
       Then query result
-        | result |
-        | 3.0    |
+        | result   |
+        |  |
 
-    Scenario: xpath_number returns a sum as double
-      When query
-        """
-        SELECT xpath_number('12', 'sum(a/b)') AS result
-        """
-      Then query result
-        | result |
-        | 3.0    |
-
-    Scenario: xpath_double returns NaN for non-numeric value
-      When query
-        """
-        SELECT xpath_double('text', 'a/b') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
+      Examples:
+        | case                                           | fn           | xml                     | path     | result |
+        | xpath_double returns a sum as double           | xpath_double | 12 | sum(a/b) | 3.0    |
+        | xpath_number returns a sum as double           | xpath_number | 12 | sum(a/b) | 3.0    |
+        | xpath_double returns NaN for non-numeric value | xpath_double | text      | a/b      | NaN    |
 
   Rule: xpath_float evaluates XPath to a float
 
-    Scenario: xpath_float returns a sum as float
+    Scenario Outline: Float: 
       When query
         """
-        SELECT xpath_float('12', 'sum(a/b)') AS result
+        SELECT xpath_float('', '') AS result
         """
       Then query result
-        | result |
-        | 3.0    |
+        | result   |
+        |  |
 
-    Scenario: xpath_float returns NaN for non-numeric value
-      When query
-        """
-        SELECT xpath_float('text', 'a/b') AS result
-        """
-      Then query result
-        | result |
-        | NaN    |
+      Examples:
+        | case                                          | xml                     | path     | result |
+        | xpath_float returns a sum as float            | 12 | sum(a/b) | 3.0    |
+        | xpath_float returns NaN for non-numeric value | text      | a/b      | NaN    |
 
   Rule: xpath_int evaluates XPath to an integer
 
-    Scenario: xpath_int returns a sum as integer
+    Scenario Outline: Int: 
       When query
         """
-        SELECT xpath_int('12', 'sum(a/b)') AS result
+        SELECT xpath_int('', '') AS result
         """
       Then query result
-        | result |
-        | 3      |
+        | result   |
+        |  |
 
-    Scenario: xpath_int returns zero for non-numeric value
-      When query
-        """
-        SELECT xpath_int('text', 'a/b') AS result
-        """
-      Then query result
-        | result |
-        | 0      |
+      Examples:
+        | case                                         | xml                     | path     | result |
+        | xpath_int returns a sum as integer           | 12 | sum(a/b) | 3      |
+        | xpath_int returns zero for non-numeric value | text      | a/b      | 0      |
 
   Rule: xpath_long evaluates XPath to a long integer
 
-    Scenario: xpath_long returns a sum as long
+    Scenario Outline: Long: 
       When query
         """
-        SELECT xpath_long('12', 'sum(a/b)') AS result
+        SELECT xpath_long('', '') AS result
         """
       Then query result
-        | result |
-        | 3      |
+        | result   |
+        |  |
 
-    Scenario: xpath_long returns zero for non-numeric value
-      When query
-        """
-        SELECT xpath_long('text', 'a/b') AS result
-        """
-      Then query result
-        | result |
-        | 0      |
+      Examples:
+        | case                                          | xml                     | path     | result |
+        | xpath_long returns a sum as long              | 12 | sum(a/b) | 3      |
+        | xpath_long returns zero for non-numeric value | text      | a/b      | 0      |
 
   Rule: xpath_short evaluates XPath to a short integer
 
-    Scenario: xpath_short returns a sum as short
+    Scenario Outline: Short: 
       When query
         """
-        SELECT xpath_short('12', 'sum(a/b)') AS result
+        SELECT xpath_short('', '') AS result
         """
       Then query result
-        | result |
-        | 3      |
+        | result   |
+        |  |
 
-    Scenario: xpath_short returns zero for non-numeric value
-      When query
-        """
-        SELECT xpath_short('text', 'a/b') AS result
-        """
-      Then query result
-        | result |
-        | 0      |
+      Examples:
+        | case                                           | xml                     | path     | result |
+        | xpath_short returns a sum as short             | 12 | sum(a/b) | 3      |
+        | xpath_short returns zero for non-numeric value | text      | a/b      | 0      |
 
   Rule: xpath_string evaluates XPath to a string
 
diff --git a/python/pysail/tests/spark/function/features/xpath.feature b/python/pysail/tests/spark/function/features/xpath.feature
deleted file mode 100644
index decb9d2f4..000000000
--- a/python/pysail/tests/spark/function/features/xpath.feature
+++ /dev/null
@@ -1,70 +0,0 @@
-Feature: xpath() extracts XML nodes with Spark-compatible semantics
-
-  Rule: Node selection returns arrays of string-like values
-
-    Scenario: xpath returns text nodes in document order
-      When query
-        """
-        SELECT xpath('b1b2b3', 'a/b/text()') AS result
-        """
-      Then query result
-        | result        |
-        | [b1, b2, b3]  |
-
-    Scenario: xpath returns NULL entries for element nodes
-      When query
-        """
-        SELECT xpath('b1b2', 'a/b') AS result
-        """
-      Then query result
-        | result        |
-        | [NULL, NULL]  |
-
-    Scenario: xpath returns an empty list when no nodes match
-      When query
-        """
-        SELECT xpath('1', 'a/c') AS result
-        """
-      Then query result
-        | result |
-        | []     |
-
-  Rule: Empty or null inputs return NULL
-
-    Scenario: xpath returns NULL for empty xml
-      When query
-        """
-        SELECT xpath('', 'a/b') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: xpath returns NULL for empty path
-      When query
-        """
-        SELECT xpath('1', '') AS result
-        """
-      Then query result
-        | result |
-        | NULL   |
-
-    Scenario: xpath returns NULL for null xml or path
-      When query
-        """
-        SELECT
-          xpath(CAST(NULL AS STRING), 'a/b') AS null_xml,
-          xpath('1', CAST(NULL AS STRING)) AS null_path
-        """
-      Then query result
-        | null_xml | null_path |
-        | NULL     | NULL      |
-
-  Rule: Non-node XPath results fail
-
-    Scenario: xpath rejects expressions that do not return a node list
-      When query
-        """
-        SELECT xpath('1', 'sum(a/b)') AS result
-        """
-      Then query error (?s).*NodeList.*
diff --git a/python/pysail/tests/spark/function/features/xxhash64.feature b/python/pysail/tests/spark/function/features/xxhash64.feature
deleted file mode 100644
index f3e8dc07a..000000000
--- a/python/pysail/tests/spark/function/features/xxhash64.feature
+++ /dev/null
@@ -1,141 +0,0 @@
-Feature: xxhash64() returns 64-bit xxHash
-
-  Rule: Basic usage
-
-    Scenario: xxhash64 integer
-      When query
-        """
-        SELECT xxhash64(42) AS result
-        """
-      Then query result
-        | result              |
-        | -387659249110444264 |
-
-    Scenario: xxhash64 string
-      When query
-        """
-        SELECT xxhash64('hello') AS result
-        """
-      Then query result
-        | result               |
-        | -4367754540140381902 |
-
-    Scenario: xxhash64 multiple args
-      When query
-        """
-        SELECT xxhash64(1, 'a', 2) AS result
-        """
-      Then query result
-        | result              |
-        | 4450643625805672383 |
-
-  Rule: Null handling
-
-    Scenario: xxhash64 null input
-      When query
-        """
-        SELECT xxhash64(CAST(NULL AS INT)) AS result
-        """
-      Then query result
-        | result |
-        | 42     |
-
-    Scenario: xxhash64 null string input also returns the seed
-      When query
-        """
-        SELECT xxhash64(CAST(NULL AS STRING)) AS result
-        """
-      Then query result
-        | result |
-        | 42     |
-
-  Rule: Type coverage
-
-    # The hash must agree with Spark for every input type, since the encoding fed
-    # to xxHash is type-specific. All values verified against the Spark JVM.
-
-    Scenario: xxhash64 bigint
-      When query
-        """
-        SELECT xxhash64(CAST(1 AS BIGINT)) AS result
-        """
-      Then query result
-        | result               |
-        | -7001672635703045582 |
-
-    Scenario: xxhash64 int and boolean true hash identically
-      When query
-        """
-        SELECT xxhash64(CAST(1 AS INT)) AS i, xxhash64(true) AS b
-        """
-      Then query result
-        | i                    | b                    |
-        | -6698625589789238999 | -6698625589789238999 |
-
-    Scenario: xxhash64 double
-      When query
-        """
-        SELECT xxhash64(CAST(1.5 AS DOUBLE)) AS result
-        """
-      Then query result
-        | result              |
-        | 7738255526519901366 |
-
-    Scenario: xxhash64 float
-      When query
-        """
-        SELECT xxhash64(CAST(1.5 AS FLOAT)) AS result
-        """
-      Then query result
-        | result              |
-        | 6163473420726370430 |
-
-    Scenario: xxhash64 decimal
-      When query
-        """
-        SELECT xxhash64(CAST(1.50 AS DECIMAL(10,2))) AS result
-        """
-      Then query result
-        | result               |
-        | -6873856301616164681 |
-
-    Scenario: xxhash64 date
-      When query
-        """
-        SELECT xxhash64(DATE '2024-01-15') AS result
-        """
-      Then query result
-        | result              |
-        | 2166432641145730595 |
-
-    # NOTE: TIMESTAMP (LTZ) is intentionally NOT asserted with a golden value —
-    # its hash depends on the session timezone (micros-since-epoch), so the value
-    # is not portable across environments. The migration still hashes it; only the
-    # golden value would be flaky.
-
-    Scenario: xxhash64 binary
-      When query
-        """
-        SELECT xxhash64(X'48656C6C6F') AS result
-        """
-      Then query result
-        | result              |
-        | 6777584228807376986 |
-
-    Scenario: xxhash64 array
-      When query
-        """
-        SELECT xxhash64(array(1, 2, 3)) AS result
-        """
-      Then query result
-        | result              |
-        | 8592097078962733837 |
-
-    Scenario: xxhash64 struct
-      When query
-        """
-        SELECT xxhash64(named_struct('a', 1, 'b', 'x')) AS result
-        """
-      Then query result
-        | result              |
-        | 8510603489595372987 |
diff --git a/python/pysail/tests/spark/function/test_arrays_zip.txt b/python/pysail/tests/spark/function/test_arrays_zip.txt
deleted file mode 100644
index 6239703b4..000000000
--- a/python/pysail/tests/spark/function/test_arrays_zip.txt
+++ /dev/null
@@ -1,55 +0,0 @@
->>> from pyspark.sql.functions import arrays_zip
->>> df = spark.createDataFrame([([1, 2, 3], [2, 4, 6], [3, 6])], ['vals1', 'vals2', 'vals3'])
->>> df = df.select(arrays_zip(df.vals1, df.vals2, df.vals3).alias('zipped'))
->>> df.show(truncate=False)
-+------------------------------------+
-|zipped                              |
-+------------------------------------+
-|[{1, 2, 3}, {2, 4, 6}, {3, 6, NULL}]|
-+------------------------------------+
-
->>> df.printSchema()
-root
- |-- zipped: array (nullable = true)
- |    |-- element: struct (containsNull = false)
- |    |    |-- vals1: long (nullable = true)
- |    |    |-- vals2: long (nullable = true)
- |    |    |-- vals3: long (nullable = true)
-
-
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame([([1, 2, 3], ['a', 'b', 'c'])], ['nums', 'letters'])
->>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False)
-+-------------------------+
-|arrays_zip(nums, letters)|
-+-------------------------+
-|[{1, a}, {2, b}, {3, c}] |
-+-------------------------+
-
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame([([1, 2], ['a', 'b', 'c'])], ['nums', 'letters'])
->>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False)
-+---------------------------+
-|arrays_zip(nums, letters)  |
-+---------------------------+
-|[{1, a}, {2, b}, {NULL, c}]|
-+---------------------------+
-
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame(
-...   [([1, 2], ['a', 'b'], [True, False])], ['nums', 'letters', 'bools'])
->>> df.select(sf.arrays_zip(df.nums, df.letters, df.bools)).show(truncate=False)
-+--------------------------------+
-|arrays_zip(nums, letters, bools)|
-+--------------------------------+
-|[{1, a, true}, {2, b, false}]   |
-+--------------------------------+
-
->>> from pyspark.sql import functions as sf
->>> df = spark.createDataFrame([([1, 2, None], ['a', None, 'c'])], ['nums', 'letters'])
->>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False)
-+------------------------------+
-|arrays_zip(nums, letters)     |
-+------------------------------+
-|[{1, a}, {2, NULL}, {NULL, c}]|
-+------------------------------+
diff --git a/python/pysail/tests/spark/function/test_bround.txt b/python/pysail/tests/spark/function/test_bround.txt
deleted file mode 100644
index c59d7af45..000000000
--- a/python/pysail/tests/spark/function/test_bround.txt
+++ /dev/null
@@ -1,17 +0,0 @@
->>> df = spark.createDataFrame([(25,),(35,)], ["n"])
->>> df.selectExpr("n", "bround(n, -1) as bround_n").show()
-+---+--------+
-|  n|bround_n|
-+---+--------+
-| 25|      20|
-| 35|      40|
-+---+--------+
-
->>> df = spark.createDataFrame([(25.0,),(35.0,)], ["n"])
->>> df.selectExpr("n", "bround(n, -1) as bround_n").show()
-+----+--------+
-|   n|bround_n|
-+----+--------+
-|25.0|    20.0|
-|35.0|    40.0|
-+----+--------+
diff --git a/python/pysail/tests/spark/function/test_ceil.txt b/python/pysail/tests/spark/function/test_ceil.txt
deleted file mode 100644
index ec85f2fb1..000000000
--- a/python/pysail/tests/spark/function/test_ceil.txt
+++ /dev/null
@@ -1,167 +0,0 @@
->>> spark.sql("SELECT ceil(-0.1), typeof(ceil(-0.1)), typeof(-0.1)").show()
-+----------+------------------+------------+
-|CEIL(-0.1)|typeof(CEIL(-0.1))|typeof(-0.1)|
-+----------+------------------+------------+
-|         0|      decimal(1,0)|decimal(1,1)|
-+----------+------------------+------------+
-
->>> spark.sql("SELECT ceil(5), typeof(ceil(5)), typeof(5)").show()
-+-------+---------------+---------+
-|CEIL(5)|typeof(CEIL(5))|typeof(5)|
-+-------+---------------+---------+
-|      5|         bigint|      int|
-+-------+---------------+---------+
-
->>> spark.sql("SELECT ceil(5.4), typeof(ceil(5.4)), typeof(5.4)").show()
-+---------+-----------------+------------+
-|CEIL(5.4)|typeof(CEIL(5.4))| typeof(5.4)|
-+---------+-----------------+------------+
-|        6|     decimal(2,0)|decimal(2,1)|
-+---------+-----------------+------------+
-
->>> spark.sql("SELECT ceil(3.1411, -3), typeof(ceil(3.1411, -3)), typeof(3.1411)").show()
-+----------------+------------------------+--------------+
-|ceil(3.1411, -3)|typeof(ceil(3.1411, -3))|typeof(3.1411)|
-+----------------+------------------------+--------------+
-|            1000|            decimal(4,0)|  decimal(5,4)|
-+----------------+------------------------+--------------+
-
->>> spark.sql("SELECT ceil(3.1411, 3), typeof(ceil(3.1411, 3)), typeof(3.1411)").show()
-+---------------+-----------------------+--------------+
-|ceil(3.1411, 3)|typeof(ceil(3.1411, 3))|typeof(3.1411)|
-+---------------+-----------------------+--------------+
-|          3.142|           decimal(5,3)|  decimal(5,4)|
-+---------------+-----------------------+--------------+
-
->>> spark.sql("SELECT ceil(3345.1, -2), typeof(ceil(3345.1, -2)), typeof(3345.1)").show()
-+----------------+------------------------+--------------+
-|ceil(3345.1, -2)|typeof(ceil(3345.1, -2))|typeof(3345.1)|
-+----------------+------------------------+--------------+
-|            3400|            decimal(5,0)|  decimal(5,1)|
-+----------------+------------------------+--------------+
-
->>> spark.sql("SELECT ceil(-12.345, 1), typeof(ceil(-12.345, 1)), typeof(3345.1)").show()
-+----------------+------------------------+--------------+
-|ceil(-12.345, 1)|typeof(ceil(-12.345, 1))|typeof(3345.1)|
-+----------------+------------------------+--------------+
-|           -12.3|            decimal(4,1)|  decimal(5,1)|
-+----------------+------------------------+--------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as TINYINT), 1), typeof(ceil(CAST(5 as TINYINT), 1))").show()
-+---------------------------+-----------------------------------+
-|ceil(CAST(5 AS TINYINT), 1)|typeof(ceil(CAST(5 AS TINYINT), 1))|
-+---------------------------+-----------------------------------+
-|                          5|                       decimal(4,0)|
-+---------------------------+-----------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as TINYINT), -4), typeof(ceil(CAST(5 as TINYINT), -4))").show()
-+----------------------------+------------------------------------+
-|ceil(CAST(5 AS TINYINT), -4)|typeof(ceil(CAST(5 AS TINYINT), -4))|
-+----------------------------+------------------------------------+
-|                       10000|                        decimal(5,0)|
-+----------------------------+------------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as SMALLINT), 1), typeof(ceil(CAST(5 as SMALLINT), 1))").show()
-+----------------------------+------------------------------------+
-|ceil(CAST(5 AS SMALLINT), 1)|typeof(ceil(CAST(5 AS SMALLINT), 1))|
-+----------------------------+------------------------------------+
-|                           5|                        decimal(6,0)|
-+----------------------------+------------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as SMALLINT), -6), typeof(ceil(CAST(5 as SMALLINT), -6))").show()
-+-----------------------------+-------------------------------------+
-|ceil(CAST(5 AS SMALLINT), -6)|typeof(ceil(CAST(5 AS SMALLINT), -6))|
-+-----------------------------+-------------------------------------+
-|                      1000000|                         decimal(7,0)|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as INT), 1), typeof(ceil(CAST(5 as INT), 1))").show()
-+-----------------------+-------------------------------+
-|ceil(CAST(5 AS INT), 1)|typeof(ceil(CAST(5 AS INT), 1))|
-+-----------------------+-------------------------------+
-|                      5|                  decimal(11,0)|
-+-----------------------+-------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as INT), -11), typeof(ceil(CAST(5 as INT), -11))").show()
-+-------------------------+---------------------------------+
-|ceil(CAST(5 AS INT), -11)|typeof(ceil(CAST(5 AS INT), -11))|
-+-------------------------+---------------------------------+
-|             100000000000|                    decimal(12,0)|
-+-------------------------+---------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as BIGINT), 1), typeof(ceil(CAST(5 as BIGINT), 1))").show()
-+--------------------------+----------------------------------+
-|ceil(CAST(5 AS BIGINT), 1)|typeof(ceil(CAST(5 AS BIGINT), 1))|
-+--------------------------+----------------------------------+
-|                         5|                     decimal(21,0)|
-+--------------------------+----------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as BIGINT), -21), typeof(ceil(CAST(5 as BIGINT), -21))").show()
-+----------------------------+------------------------------------+
-|ceil(CAST(5 AS BIGINT), -21)|typeof(ceil(CAST(5 AS BIGINT), -21))|
-+----------------------------+------------------------------------+
-|        10000000000000000...|                       decimal(22,0)|
-+----------------------------+------------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as FLOAT), 1), typeof(ceil(CAST(5 as FLOAT), 1))").show()
-+-------------------------+---------------------------------+
-|ceil(CAST(5 AS FLOAT), 1)|typeof(ceil(CAST(5 AS FLOAT), 1))|
-+-------------------------+---------------------------------+
-|                      5.0|                     decimal(9,1)|
-+-------------------------+---------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as FLOAT), -15), typeof(ceil(CAST(5 as FLOAT), -15))").show()
-+---------------------------+-----------------------------------+
-|ceil(CAST(5 AS FLOAT), -15)|typeof(ceil(CAST(5 AS FLOAT), -15))|
-+---------------------------+-----------------------------------+
-|           1000000000000000|                      decimal(16,0)|
-+---------------------------+-----------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as DOUBLE), 1), typeof(ceil(CAST(5 as DOUBLE), 1))").show()
-+--------------------------+----------------------------------+
-|ceil(CAST(5 AS DOUBLE), 1)|typeof(ceil(CAST(5 AS DOUBLE), 1))|
-+--------------------------+----------------------------------+
-|                       5.0|                     decimal(17,1)|
-+--------------------------+----------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as DOUBLE), -31), typeof(ceil(CAST(5 as DOUBLE), -31))").show()
-+----------------------------+------------------------------------+
-|ceil(CAST(5 AS DOUBLE), -31)|typeof(ceil(CAST(5 AS DOUBLE), -31))|
-+----------------------------+------------------------------------+
-|        10000000000000000...|                       decimal(32,0)|
-+----------------------------+------------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as DECIMAL(5, 2)), 1), typeof(ceil(CAST(5 as DECIMAL(5, 2)), 1))").show()
-+--------------------------------+----------------------------------------+
-|ceil(CAST(5 AS DECIMAL(5,2)), 1)|typeof(ceil(CAST(5 AS DECIMAL(5,2)), 1))|
-+--------------------------------+----------------------------------------+
-|                             5.0|                            decimal(5,1)|
-+--------------------------------+----------------------------------------+
-
->>> spark.sql("SELECT ceil(CAST(5 as DECIMAL(5, 2))), typeof(ceil(CAST(5 as DECIMAL(5, 2))))").show()
-+-----------------------------+-------------------------------------+
-|CEIL(CAST(5 AS DECIMAL(5,2)))|typeof(CEIL(CAST(5 AS DECIMAL(5,2))))|
-+-----------------------------+-------------------------------------+
-|                            5|                         decimal(4,0)|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT ceil(5.4, -1), typeof(ceil(5.4, -1)), typeof(5.4)").show()
-+-------------+---------------------+------------+
-|ceil(5.4, -1)|typeof(ceil(5.4, -1))| typeof(5.4)|
-+-------------+---------------------+------------+
-|           10|         decimal(2,0)|decimal(2,1)|
-+-------------+---------------------+------------+
-
->>> spark.sql("SELECT ceil(5, -1), typeof(ceil(5, -1)), typeof(5)").show()
-+-----------+-------------------+---------+
-|ceil(5, -1)|typeof(ceil(5, -1))|typeof(5)|
-+-----------+-------------------+---------+
-|         10|      decimal(11,0)|      int|
-+-----------+-------------------+---------+
-
->>> spark.sql("SELECT ceil(5, 0), typeof(ceil(5, 0)), typeof(5)").show()
-+----------+------------------+---------+
-|ceil(5, 0)|typeof(ceil(5, 0))|typeof(5)|
-+----------+------------------+---------+
-|         5|     decimal(11,0)|      int|
-+----------+------------------+---------+
diff --git a/python/pysail/tests/spark/function/test_concat_ws.txt b/python/pysail/tests/spark/function/test_concat_ws.txt
deleted file mode 100644
index d39c5af42..000000000
--- a/python/pysail/tests/spark/function/test_concat_ws.txt
+++ /dev/null
@@ -1,19 +0,0 @@
->>> from datetime import date
->>> from pyspark.sql import functions as F, types as T
-
->>> schema = T.StructType([
-...     T.StructField("string_col", T.StringType(), True),
-...     T.StructField("int_col", T.IntegerType(), True),
-...     T.StructField("date_col", T.DateType(), True),
-... ])
->>> df = spark.createDataFrame([("a", 1, date(2024, 1, 15))], schema)
->>> df.select(F.concat_ws("-", F.col("string_col"), F.col("int_col"), F.col("date_col"))).collect()
-[Row(concat_ws(-, string_col, int_col, date_col)='a-1-2024-01-15')]
-
->>> schema = T.StructType([
-...     T.StructField("string_col", T.StringType(), True),
-...     T.StructField("int_col", T.IntegerType(), True),
-... ])
->>> df = spark.createDataFrame([("a", None)], schema)
->>> df.select(F.concat_ws("-", F.col("string_col"), F.col("int_col"))).collect()
-[Row(concat_ws(-, string_col, int_col)='a')]
\ No newline at end of file
diff --git a/python/pysail/tests/spark/function/test_conv.txt b/python/pysail/tests/spark/function/test_conv.txt
deleted file mode 100644
index 6c0cd2572..000000000
--- a/python/pysail/tests/spark/function/test_conv.txt
+++ /dev/null
@@ -1,7 +0,0 @@
->>> df = spark.sql("""SELECT conv('10', 10, 2) as to_binary,conv('10', 10, 8) as to_octal,conv('10', 10, 16) as to_hex""")
->>> df.show()
-+---------+--------+------+
-|to_binary|to_octal|to_hex|
-+---------+--------+------+
-|     1010|      12|     A|
-+---------+--------+------+
diff --git a/python/pysail/tests/spark/function/test_csc.txt b/python/pysail/tests/spark/function/test_csc.txt
deleted file mode 100644
index c86569f28..000000000
--- a/python/pysail/tests/spark/function/test_csc.txt
+++ /dev/null
@@ -1,45 +0,0 @@
->>> from pyspark.sql import functions as sf
->>> import math
-
->>> spark.sql("SELECT csc(radians(90)) AS csc").show()
-+---+
-|csc|
-+---+
-|1.0|
-+---+
-
->>> df = spark.range(1)
->>> df.select(sf.csc(sf.lit(math.radians(90))).alias("csc")).show()
-+---+
-|csc|
-+---+
-|1.0|
-+---+
-
->>> df.select(sf.csc(sf.lit(90)).alias("csc")).show()
-+------------------+
-|               csc|
-+------------------+
-|1.1185724071637084|
-+------------------+
-
->>> df.select(sf.csc(sf.lit(90.0)).alias("csc")).show()
-+------------------+
-|               csc|
-+------------------+
-|1.1185724071637084|
-+------------------+
-
->>> df.select(sf.csc(sf.lit(-90.0)).alias("csc")).show()
-+-------------------+
-|                csc|
-+-------------------+
-|-1.1185724071637084|
-+-------------------+
-
->>> df.select(sf.csc(sf.lit(0)).alias("csc")).show()
-+--------+
-|     csc|
-+--------+
-|Infinity|
-+--------+
diff --git a/python/pysail/tests/spark/function/test_elt.txt b/python/pysail/tests/spark/function/test_elt.txt
deleted file mode 100644
index 7017343a1..000000000
--- a/python/pysail/tests/spark/function/test_elt.txt
+++ /dev/null
@@ -1,33 +0,0 @@
->>> spark.sql("SELECT elt(1, 10, 20)").printSchema()
-root
- |-- elt(1, 10, 20): string (nullable = true)
-
->>> spark.sql("SELECT elt(1, 10, 20)").show()
-+--------------+
-|elt(1, 10, 20)|
-+--------------+
-|            10|
-+--------------+
-
->>> spark.sql("SELECT elt(1, 'scala', 'java')").show()
-+-------------------+
-|elt(1, scala, java)|
-+-------------------+
-|              scala|
-+-------------------+
-
->>> spark.sql("SELECT elt(11, 10, 20)").show()
-+---------------+
-|elt(11, 10, 20)|
-+---------------+
-|           NULL|
-+---------------+
-
->>> spark.sql("SELECT elt(6, 'scala', 'java', 'c', 'c++', 'python', 'rust')").show()
-+-----------------------------------------+
-|elt(6, scala, java, c, c++, python, rust)|
-+-----------------------------------------+
-|                                     rust|
-+-----------------------------------------+
-
-# If spark.sql.ansi.enabled is set to true, it throws ArrayIndexOutOfBoundsException for invalid indices.
diff --git a/python/pysail/tests/spark/function/test_floor.txt b/python/pysail/tests/spark/function/test_floor.txt
deleted file mode 100644
index eabf021a4..000000000
--- a/python/pysail/tests/spark/function/test_floor.txt
+++ /dev/null
@@ -1,167 +0,0 @@
->>> spark.sql("SELECT floor(-0.1), typeof(floor(-0.1)), typeof(-0.1)").show()
-+-----------+-------------------+------------+
-|FLOOR(-0.1)|typeof(FLOOR(-0.1))|typeof(-0.1)|
-+-----------+-------------------+------------+
-|         -1|       decimal(1,0)|decimal(1,1)|
-+-----------+-------------------+------------+
-
->>> spark.sql("SELECT floor(5), typeof(floor(5)), typeof(5)").show()
-+--------+----------------+---------+
-|FLOOR(5)|typeof(FLOOR(5))|typeof(5)|
-+--------+----------------+---------+
-|       5|          bigint|      int|
-+--------+----------------+---------+
-
->>> spark.sql("SELECT floor(5.4), typeof(floor(5.4)), typeof(5.4)").show()
-+----------+------------------+------------+
-|FLOOR(5.4)|typeof(FLOOR(5.4))| typeof(5.4)|
-+----------+------------------+------------+
-|         5|      decimal(2,0)|decimal(2,1)|
-+----------+------------------+------------+
-
->>> spark.sql("SELECT floor(3.1411, -3), typeof(floor(3.1411, -3)), typeof(3.1411)").show()
-+-----------------+-------------------------+--------------+
-|floor(3.1411, -3)|typeof(floor(3.1411, -3))|typeof(3.1411)|
-+-----------------+-------------------------+--------------+
-|                0|             decimal(4,0)|  decimal(5,4)|
-+-----------------+-------------------------+--------------+
-
->>> spark.sql("SELECT floor(3.1411, 3), typeof(floor(3.1411, 3)), typeof(3.1411)").show()
-+----------------+------------------------+--------------+
-|floor(3.1411, 3)|typeof(floor(3.1411, 3))|typeof(3.1411)|
-+----------------+------------------------+--------------+
-|           3.141|            decimal(5,3)|  decimal(5,4)|
-+----------------+------------------------+--------------+
-
->>> spark.sql("SELECT floor(3345.1, -2), typeof(floor(3345.1, -2)), typeof(3345.1)").show()
-+-----------------+-------------------------+--------------+
-|floor(3345.1, -2)|typeof(floor(3345.1, -2))|typeof(3345.1)|
-+-----------------+-------------------------+--------------+
-|             3300|             decimal(5,0)|  decimal(5,1)|
-+-----------------+-------------------------+--------------+
-
->>> spark.sql("SELECT floor(-12.345, 1), typeof(floor(-12.345, 1)), typeof(3345.1)").show()
-+-----------------+-------------------------+--------------+
-|floor(-12.345, 1)|typeof(floor(-12.345, 1))|typeof(3345.1)|
-+-----------------+-------------------------+--------------+
-|            -12.4|             decimal(4,1)|  decimal(5,1)|
-+-----------------+-------------------------+--------------+
-
->>> spark.sql("SELECT floor(CAST(5 as TINYINT), 1), typeof(floor(CAST(5 as TINYINT), 1))").show()
-+----------------------------+------------------------------------+
-|floor(CAST(5 AS TINYINT), 1)|typeof(floor(CAST(5 AS TINYINT), 1))|
-+----------------------------+------------------------------------+
-|                           5|                        decimal(4,0)|
-+----------------------------+------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as TINYINT), -4), typeof(floor(CAST(5 as TINYINT), -4))").show()
-+-----------------------------+-------------------------------------+
-|floor(CAST(5 AS TINYINT), -4)|typeof(floor(CAST(5 AS TINYINT), -4))|
-+-----------------------------+-------------------------------------+
-|                            0|                         decimal(5,0)|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as SMALLINT), 1), typeof(floor(CAST(5 as SMALLINT), 1))").show()
-+-----------------------------+-------------------------------------+
-|floor(CAST(5 AS SMALLINT), 1)|typeof(floor(CAST(5 AS SMALLINT), 1))|
-+-----------------------------+-------------------------------------+
-|                            5|                         decimal(6,0)|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as SMALLINT), -6), typeof(floor(CAST(5 as SMALLINT), -6))").show()
-+------------------------------+--------------------------------------+
-|floor(CAST(5 AS SMALLINT), -6)|typeof(floor(CAST(5 AS SMALLINT), -6))|
-+------------------------------+--------------------------------------+
-|                             0|                          decimal(7,0)|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as INT), 1), typeof(floor(CAST(5 as INT), 1))").show()
-+------------------------+--------------------------------+
-|floor(CAST(5 AS INT), 1)|typeof(floor(CAST(5 AS INT), 1))|
-+------------------------+--------------------------------+
-|                       5|                   decimal(11,0)|
-+------------------------+--------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as INT), -11), typeof(floor(CAST(5 as INT), -11))").show()
-+--------------------------+----------------------------------+
-|floor(CAST(5 AS INT), -11)|typeof(floor(CAST(5 AS INT), -11))|
-+--------------------------+----------------------------------+
-|                         0|                     decimal(12,0)|
-+--------------------------+----------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as BIGINT), 1), typeof(floor(CAST(5 as BIGINT), 1))").show()
-+---------------------------+-----------------------------------+
-|floor(CAST(5 AS BIGINT), 1)|typeof(floor(CAST(5 AS BIGINT), 1))|
-+---------------------------+-----------------------------------+
-|                          5|                      decimal(21,0)|
-+---------------------------+-----------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as BIGINT), -21), typeof(floor(CAST(5 as BIGINT), -21))").show()
-+-----------------------------+-------------------------------------+
-|floor(CAST(5 AS BIGINT), -21)|typeof(floor(CAST(5 AS BIGINT), -21))|
-+-----------------------------+-------------------------------------+
-|                            0|                        decimal(22,0)|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as FLOAT), 1), typeof(floor(CAST(5 as FLOAT), 1))").show()
-+--------------------------+----------------------------------+
-|floor(CAST(5 AS FLOAT), 1)|typeof(floor(CAST(5 AS FLOAT), 1))|
-+--------------------------+----------------------------------+
-|                       5.0|                      decimal(9,1)|
-+--------------------------+----------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as FLOAT), -15), typeof(floor(CAST(5 as FLOAT), -15))").show()
-+----------------------------+------------------------------------+
-|floor(CAST(5 AS FLOAT), -15)|typeof(floor(CAST(5 AS FLOAT), -15))|
-+----------------------------+------------------------------------+
-|                           0|                       decimal(16,0)|
-+----------------------------+------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as DOUBLE), 1), typeof(floor(CAST(5 as DOUBLE), 1))").show()
-+---------------------------+-----------------------------------+
-|floor(CAST(5 AS DOUBLE), 1)|typeof(floor(CAST(5 AS DOUBLE), 1))|
-+---------------------------+-----------------------------------+
-|                        5.0|                      decimal(17,1)|
-+---------------------------+-----------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as DOUBLE), -31), typeof(floor(CAST(5 as DOUBLE), -31))").show()
-+-----------------------------+-------------------------------------+
-|floor(CAST(5 AS DOUBLE), -31)|typeof(floor(CAST(5 AS DOUBLE), -31))|
-+-----------------------------+-------------------------------------+
-|                            0|                        decimal(32,0)|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as DECIMAL(5, 2)), 1), typeof(floor(CAST(5 as DECIMAL(5, 2)), 1))").show()
-+---------------------------------+-----------------------------------------+
-|floor(CAST(5 AS DECIMAL(5,2)), 1)|typeof(floor(CAST(5 AS DECIMAL(5,2)), 1))|
-+---------------------------------+-----------------------------------------+
-|                              5.0|                             decimal(5,1)|
-+---------------------------------+-----------------------------------------+
-
->>> spark.sql("SELECT floor(CAST(5 as DECIMAL(5, 2))), typeof(floor(CAST(5 as DECIMAL(5, 2))))").show()
-+------------------------------+--------------------------------------+
-|FLOOR(CAST(5 AS DECIMAL(5,2)))|typeof(FLOOR(CAST(5 AS DECIMAL(5,2))))|
-+------------------------------+--------------------------------------+
-|                             5|                          decimal(4,0)|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT floor(5.4, -1), typeof(floor(5.4, -1)), typeof(5.4)").show()
-+--------------+----------------------+------------+
-|floor(5.4, -1)|typeof(floor(5.4, -1))| typeof(5.4)|
-+--------------+----------------------+------------+
-|             0|          decimal(2,0)|decimal(2,1)|
-+--------------+----------------------+------------+
-
->>> spark.sql("SELECT floor(5, -1), typeof(floor(5, -1)), typeof(5)").show()
-+------------+--------------------+---------+
-|floor(5, -1)|typeof(floor(5, -1))|typeof(5)|
-+------------+--------------------+---------+
-|           0|       decimal(11,0)|      int|
-+------------+--------------------+---------+
-
->>> spark.sql("SELECT floor(5, 0), typeof(floor(5, 0)), typeof(5)").show()
-+-----------+-------------------+---------+
-|floor(5, 0)|typeof(floor(5, 0))|typeof(5)|
-+-----------+-------------------+---------+
-|          5|      decimal(11,0)|      int|
-+-----------+-------------------+---------+
diff --git a/python/pysail/tests/spark/function/test_from_csv.txt b/python/pysail/tests/spark/function/test_from_csv.txt
deleted file mode 100644
index 55a382256..000000000
--- a/python/pysail/tests/spark/function/test_from_csv.txt
+++ /dev/null
@@ -1,35 +0,0 @@
->>> spark.sql("SELECT from_csv('1, 0.8', 'a INT, b DOUBLE') AS result, typeof(from_csv('1, 0.8', 'a INT, b DOUBLE')) AS type").show(truncate=False)
-+--------+----------------------+
-|result  |type                  |
-+--------+----------------------+
-|{1, 0.8}|struct|
-+--------+----------------------+
-
->>> spark.sql("SELECT from_csv('2015-08-26 00:00:00', 'time TIMESTAMP') AS result, typeof(from_csv('2015-08-26 00:00:00', 'time TIMESTAMP')) AS type").show(truncate=False)
-+---------------------+----------------------+
-|result               |type                  |
-+---------------------+----------------------+
-|{2015-08-26 00:00:00}|struct|
-+---------------------+----------------------+
-
->>> spark.sql("SELECT from_csv('42;3.14;2024-12-01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';')) AS result, typeof(from_csv('42;3.14;2024-12-01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';'))) AS type").show(truncate=False)
-+-------------------------------+-----------------------------------------------+
-|result                         |type                                           |
-+-------------------------------+-----------------------------------------------+
-|{42, 3.14, 2024-12-01 10:00:00}|struct|
-+-------------------------------+-----------------------------------------------+
-
->>> spark.sql("SELECT from_csv('42;3.14;2024/12/01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';', 'timestampFormat', 'yyyy/MM/dd HH:mm:ss')) AS result, typeof(from_csv('42;3.14;2024/12/01 10:00:00', 'id INT, value DOUBLE, timestamp TIMESTAMP', map('sep', ';', 'timestampFormat', 'yyyy/MM/dd HH:mm:ss'))) AS type").show(truncate=False)
-+-------------------------------+-----------------------------------------------+
-|result                         |type                                           |
-+-------------------------------+-----------------------------------------------+
-|{42, 3.14, 2024-12-01 10:00:00}|struct|
-+-------------------------------+-----------------------------------------------+
-
-
->>> spark.sql("SELECT from_csv('26/08/2015', 'time TIMESTAMP', map('timestampFormat', 'dd/MM/yyyy')) AS result, typeof(from_csv('26/08/2015', 'time TIMESTAMP', map('timestampFormat', 'dd/MM/yyyy'))) AS type").show(truncate=False)
-+---------------------+----------------------+
-|result               |type                  |
-+---------------------+----------------------+
-|{2015-08-26 00:00:00}|struct|
-+---------------------+----------------------+
diff --git a/python/pysail/tests/spark/function/test_from_json.txt b/python/pysail/tests/spark/function/test_from_json.txt
deleted file mode 100644
index 930ed4ca6..000000000
--- a/python/pysail/tests/spark/function/test_from_json.txt
+++ /dev/null
@@ -1,72 +0,0 @@
->>> from pyspark.sql.functions import from_json
->>> from pyspark.sql.types import *
-
->>> data = [(1, '''{"a": 1}''')]
->>> schema = StructType([StructField("a", IntegerType())])
->>> df = spark.createDataFrame(data, ("key", "value"))
->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False)
-+----+
-|json|
-+----+
-|{1} |
-+----+
-
-
->>> df.select(from_json(df.value, "a INT").alias("json")).show(truncate=False)
-+----+
-|json|
-+----+
-|{1} |
-+----+
-
-
->>> df.select(from_json(df.value, "MAP").alias("json")).show(truncate=False)
-+--------+
-|json    |
-+--------+
-|{a -> 1}|
-+--------+
-
-
->>> schema = ArrayType(StructType([StructField("a", IntegerType())]))
->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False)
-+-----+
-|json |
-+-----+
-|[{1}]|
-+-----+
-
-
->>> data = [(1, '''{"a": 1}'''), (2, '''{"a": 2}'''), (3, None)]
->>> df = spark.createDataFrame(data, ("key", "value"))
->>> df.select(from_json(df.value, "a INT").alias("json")).orderBy("key").show(truncate=False)
-+----+
-|json|
-+----+
-|{1} |
-|{2} |
-|NULL|
-+----+
-
-
->>> data = [(1, '''{"a": 1, "b": "hello"}''')]
->>> df = spark.createDataFrame(data, ("key", "value"))
->>> schema = StructType([StructField("a", IntegerType()), StructField("b", StringType())])
->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False)
-+----------+
-|json      |
-+----------+
-|{1, hello}|
-+----------+
-
-
->>> data = [(1, '''{"price": 19.99}''')]
->>> df = spark.createDataFrame(data, ("key", "value"))
->>> schema = StructType([StructField("price", DecimalType(10, 2))])
->>> df.select(from_json(df.value, schema).alias("json")).show(truncate=False)
-+-------+
-|json   |
-+-------+
-|{19.99}|
-+-------+
-
diff --git a/python/pysail/tests/spark/function/test_get_json_object.txt b/python/pysail/tests/spark/function/test_get_json_object.txt
deleted file mode 100644
index 2b5d69341..000000000
--- a/python/pysail/tests/spark/function/test_get_json_object.txt
+++ /dev/null
@@ -1,27 +0,0 @@
->>> spark.sql("""SELECT
-...   get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.a') AS a,
-...   get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.b') AS b,
-...   get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.c') AS c,
-...   get_json_object('{"a":"x","b":1,"c":{"d":true},"e":[10,20]}', '$.e') AS e
-... """).show(truncate=False)
-+---+---+----------+-------+
-|a  |b  |c         |e      |
-+---+---+----------+-------+
-|x  |1  |{"d":true}|[10,20]|
-+---+---+----------+-------+
-
->>> spark.sql("""SELECT
-...   get_json_object('{"a":null}', '$.a') AS null_value,
-...   get_json_object('{"a":1}', '$.missing') AS missing_key,
-...   get_json_object(CAST(NULL AS STRING), '$.a') AS null_json,
-...   get_json_object('{"a":1}', CAST(NULL AS STRING)) AS null_path,
-...   get_json_object('not a json', '$.a') AS invalid_json,
-...   get_json_object('{"a":1}', 'a') AS invalid_path
-... """).show(truncate=False)
-+----------+-----------+---------+---------+------------+------------+
-|null_value|missing_key|null_json|null_path|invalid_json|invalid_path|
-+----------+-----------+---------+---------+------------+------------+
-|NULL      |NULL       |NULL     |NULL     |NULL        |NULL        |
-+----------+-----------+---------+---------+------------+------------+
-
-
diff --git a/python/pysail/tests/spark/function/test_input_order.py b/python/pysail/tests/spark/function/test_input_order.py
new file mode 100644
index 000000000..c7204f2a1
--- /dev/null
+++ b/python/pysail/tests/spark/function/test_input_order.py
@@ -0,0 +1,123 @@
+"""Input-order preservation cases that are NOT expressible as SQL feature scenarios.
+
+The SQL-expressible majority of the old ``test_input_order.txt`` doctests now lives in
+``features/aggregate/input_order.feature``. This module keeps the cases that need the
+DataFrame API, Python UDFs, file I/O, partitioning, or plan inspection — plus one known
+Sail-vs-Spark divergence (floating-point summation order).
+
+Note: the ``pandas_udf`` tests require the Python UDF worker, which does not run against a
+local Sail server (they pass against JVM Spark, matching ``udf/test_pandas_agg_window.py``).
+"""
+
+import re
+import tempfile
+
+import pandas as pd
+import pytest
+from pyspark.sql import Window
+from pyspark.sql import functions as F  # noqa: N812
+from pyspark.sql.functions import pandas_udf
+
+from pysail.testing.spark.utils.common import is_jvm_spark
+
+
+@pytest.mark.skipif(
+    not is_jvm_spark(),
+    reason="requires the Python UDF worker (JVM Spark or a sail-server-python launch)",
+)
+def test_pandas_udf_first_arrival_preserves_order(spark):
+    df = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k"))
+
+    @pandas_udf("long")
+    def first_arrival(v: pd.Series) -> int:
+        return int(v.iloc[0])
+
+    result = df.orderBy("k").groupBy().agg(first_arrival("v").alias("fa")).collect()
+    assert [tuple(r) for r in result] == [(10,)]
+
+
+@pytest.mark.skipif(
+    not is_jvm_spark(),
+    reason="requires the Python UDF worker (JVM Spark or a sail-server-python launch)",
+)
+def test_pandas_udf_first_arrival_stable_coalesce(spark):
+    stable = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k")).coalesce(1)
+
+    @pandas_udf("long")
+    def first_arrival(v: pd.Series) -> int:
+        return int(v.iloc[0])
+
+    result = stable.orderBy("k").groupBy().agg(first_arrival("v").alias("fa")).collect()
+    assert [tuple(r) for r in result] == [(10,)]
+
+
+def test_repartition_then_order_preserved(spark):
+    dft = spark.createDataFrame([(1, "a"), (2, "a"), (3, "a")], ("v", "k")).repartition(3)
+    result = dft.orderBy("k").groupBy().agg(F.first("v")).collect()
+    assert [tuple(r) for r in result] == [(1,)]
+
+
+def test_coalesce_window_sees_input_order(spark):
+    df = spark.createDataFrame([(30, "c", "x"), (10, "a", "x"), (20, "b", "x")], ("v", "k", "g")).coalesce(1)
+    w_desc = Window.partitionBy("g").orderBy(F.desc("k"))
+    w_none = Window.partitionBy("g")
+    result = df.orderBy("k").select("v", "k", F.sum("v").over(w_desc).alias("s"), F.last("v").over(w_none).alias("l"))
+    got = sorted((r.v, r.k, r.s, r.l) for r in result.collect())
+    assert got == [(10, "a", 60, 10), (20, "b", 50, 10), (30, "c", 30, 10)]
+
+
+def test_projected_column_order_survives_parquet_roundtrip(spark):
+    df = spark.createDataFrame([(30, "c", "x"), (10, "a", "x"), (20, "b", "x")], ("v", "k", "g")).coalesce(1)
+    w_desc = Window.partitionBy("g").orderBy(F.desc("k"))
+    w_none = Window.partitionBy("g")
+    result = df.orderBy("k").select("*", F.sum("v").over(w_desc).alias("s"), F.last("v").over(w_none).alias("l"))
+    assert result.columns == ["v", "k", "g", "s", "l"]
+    assert result.select("*").columns == ["v", "k", "g", "s", "l"]
+    with tempfile.TemporaryDirectory() as path:
+        result.write.mode("overwrite").parquet(path)
+        written_columns = spark.read.parquet(path).columns
+    assert written_columns == ["v", "k", "g", "s", "l"]
+
+
+def test_monotonically_increasing_id_does_not_disturb_order(spark):
+    df = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k"))
+    result = df.orderBy("k").withColumn("i", F.monotonically_increasing_id()).groupBy().agg(F.first("v")).collect()
+    assert [tuple(r) for r in result] == [(10,)]
+
+
+def test_spark_partition_id_does_not_disturb_order(spark):
+    df = spark.createDataFrame([(30, "c"), (10, "a"), (20, "b")], ("v", "k"))
+    result = df.orderBy("k").withColumn("p", F.spark_partition_id()).groupBy().agg(F.first("v")).collect()
+    assert [tuple(r) for r in result] == [(10,)]
+
+
+def test_pivot_first_preserves_order(spark):
+    pdf = spark.createDataFrame([(30, "c", "x"), (10, "a", "x"), (20, "b", "y")], ("v", "k", "p"))
+    result = pdf.orderBy("k").groupBy().pivot("p").agg(F.first("v")).collect()
+    assert [tuple(r) for r in result] == [(10, 20)]
+
+
+def test_first_is_not_rewritten_with_an_order_by_in_the_plan(spark):
+    explain = spark.sql(
+        "EXPLAIN EXTENDED SELECT first(v) FROM (SELECT * FROM VALUES (30,'c'),(10,'a'),(20,'b') AS t(v, k) ORDER BY k)"
+    ).first()[0]
+    assert re.search(r"first(?:_value)?\([^\n]* ORDER BY \[", explain, re.IGNORECASE) is None
+
+
+@pytest.mark.skipif(
+    not is_jvm_spark(),
+    reason="DataFusion reduces a batch with tree/SIMD summation, so the floating-point "
+    "skewness of these near-cancelling values is unstable around 0 on Sail (see the FIXME "
+    "in the original doctest). JVM Spark sums sequentially and yields a stable non-zero value.",
+)
+def test_skewness_is_order_of_summation_dependent(spark):
+    sk = (
+        spark.sql(
+            "SELECT skewness(v) AS sk FROM (SELECT * FROM VALUES "
+            "(CAST(1e16 AS DOUBLE),'b'),(CAST(1.0 AS DOUBLE),'c'),(CAST(-1e16 AS DOUBLE),'a') "
+            "AS t(v, k) ORDER BY k)"
+        )
+        .first()
+        .sk
+    )
+    assert sk != 0.0
diff --git a/python/pysail/tests/spark/function/test_json_array_length.txt b/python/pysail/tests/spark/function/test_json_array_length.txt
deleted file mode 100644
index 8cbbdc2cc..000000000
--- a/python/pysail/tests/spark/function/test_json_array_length.txt
+++ /dev/null
@@ -1,26 +0,0 @@
->>> spark.sql("SELECT json_array_length('[1,2,3]') AS arr_len, typeof(json_array_length('[1,2,3]')) AS type").show(truncate=False)
-+-------+----+
-|arr_len|type|
-+-------+----+
-|3      |int |
-+-------+----+
-
->>> spark.sql("SELECT json_array_length('[]') AS arr_len, typeof(json_array_length('[]')) AS type").show(truncate=False)
-+-------+----+
-|arr_len|type|
-+-------+----+
-|0      |int |
-+-------+----+
-
->>> spark.sql("""SELECT
-...   json_array_length(CAST(NULL AS STRING)) AS null_json,
-...   json_array_length('not a json') AS invalid_json,
-...   json_array_length('1') AS scalar_json
-... """).show(truncate=False)
-+---------+------------+-----------+
-|null_json|invalid_json|scalar_json|
-+---------+------------+-----------+
-|NULL     |NULL        |NULL       |
-+---------+------------+-----------+
-
-
diff --git a/python/pysail/tests/spark/function/test_json_object_keys.txt b/python/pysail/tests/spark/function/test_json_object_keys.txt
deleted file mode 100644
index b7bb05317..000000000
--- a/python/pysail/tests/spark/function/test_json_object_keys.txt
+++ /dev/null
@@ -1,31 +0,0 @@
->>> spark.sql("""SELECT
-...   json_object_keys('{"f1":"abc","f2":{"f3":"a","f4":"b"}}') AS keys,
-...   typeof(json_object_keys('{"f1":"abc","f2":{"f3":"a","f4":"b"}}')) AS type
-... """).show(truncate=False)
-+--------+-------------+
-|keys    |type         |
-+--------+-------------+
-|[f1, f2]|array|
-+--------+-------------+
-
-
->>> spark.sql("SELECT json_object_keys('{}') AS keys, typeof(json_object_keys('{}')) AS type").show(truncate=False)
-+----+-------------+
-|keys|type         |
-+----+-------------+
-|[]  |array|
-+----+-------------+
-
-
->>> spark.sql("""SELECT
-...   json_object_keys('[1,2]') AS non_object,
-...   json_object_keys(CAST(NULL AS STRING)) AS null_json,
-...   json_object_keys('not a json') AS invalid_json
-... """).show(truncate=False)
-+----------+---------+------------+
-|non_object|null_json|invalid_json|
-+----------+---------+------------+
-|NULL      |NULL     |NULL        |
-+----------+---------+------------+
-
-
diff --git a/python/pysail/tests/spark/function/test_make_dt_interval.txt b/python/pysail/tests/spark/function/test_make_dt_interval.txt
deleted file mode 100644
index 493d99faf..000000000
--- a/python/pysail/tests/spark/function/test_make_dt_interval.txt
+++ /dev/null
@@ -1,124 +0,0 @@
-# null
->>> spark.sql("SELECT (make_dt_interval(null, 0, 0, 0))").show(truncate=False)
-+-------------------------------+
-|make_dt_interval(NULL, 0, 0, 0)|
-+-------------------------------+
-|NULL                           |
-+-------------------------------+
->>> spark.sql("SELECT (make_dt_interval(0, null, 0, 0))").show(truncate=False)
-+-------------------------------+
-|make_dt_interval(0, NULL, 0, 0)|
-+-------------------------------+
-|NULL                           |
-+-------------------------------+
->>> spark.sql("SELECT (make_dt_interval(0, 0, null, 0))").show(truncate=False)
-+-------------------------------+
-|make_dt_interval(0, 0, NULL, 0)|
-+-------------------------------+
-|NULL                           |
-+-------------------------------+
->>> spark.sql("SELECT (make_dt_interval(0, 0, 0, null))").show(truncate=False)
-+-------------------------------+
-|make_dt_interval(0, 0, 0, NULL)|
-+-------------------------------+
-|NULL                           |
-+-------------------------------+
-
-# missing params
->>> spark.sql("SELECT (make_dt_interval()) AS make_dt_interval").show(truncate=False)
-+-----------------------------------+
-|make_dt_interval                   |
-+-----------------------------------+
-|INTERVAL '0 00:00:00' DAY TO SECOND|
-+-----------------------------------+
->>> spark.sql("SELECT (make_dt_interval(1)) AS make_dt_interval").show(truncate=False)
-+-----------------------------------+
-|make_dt_interval                   |
-+-----------------------------------+
-|INTERVAL '1 00:00:00' DAY TO SECOND|
-+-----------------------------------+
->>> spark.sql("SELECT (make_dt_interval(1, 1)) AS make_dt_interval").show(truncate=False)
-+-----------------------------------+
-|make_dt_interval                   |
-+-----------------------------------+
-|INTERVAL '1 01:00:00' DAY TO SECOND|
-+-----------------------------------+
->>> spark.sql("SELECT (make_dt_interval(1, 1, 1)) AS make_dt_interval").show(truncate=False)
-+-----------------------------------+
-|make_dt_interval                   |
-+-----------------------------------+
-|INTERVAL '1 01:01:00' DAY TO SECOND|
-+-----------------------------------+
->>> spark.sql("SELECT (make_dt_interval(1, 1, 1, 1)) AS make_dt_interval").show(truncate=False)
-+-----------------------------------+
-|make_dt_interval                   |
-+-----------------------------------+
-|INTERVAL '1 01:01:01' DAY TO SECOND|
-+-----------------------------------+
-
-# all 0 values
->>> spark.sql("SELECT (make_dt_interval(0, 0, 0, 0))").show(truncate=False)
-+-----------------------------------+
-|make_dt_interval(0, 0, 0, 0)       |
-+-----------------------------------+
-|INTERVAL '0 00:00:00' DAY TO SECOND|
-+-----------------------------------+
-
->>> spark.sql("SELECT (make_dt_interval(-1, 24, 0, 0)) df").show(truncate=False)
-+-----------------------------------+
-|df                                 |
-+-----------------------------------+
-|INTERVAL '0 00:00:00' DAY TO SECOND|
-+-----------------------------------+
->>> spark.sql("SELECT (make_dt_interval(1, -24, 0, 0)) dt").show(truncate=False)
-+-----------------------------------+
-|dt                                 |
-+-----------------------------------+
-|INTERVAL '0 00:00:00' DAY TO SECOND|
-+-----------------------------------+
-
->>> spark.sql("SELECT (make_dt_interval(0, 0, 0, 0.1))").show(truncate=False)
-+-------------------------------------+
-|make_dt_interval(0, 0, 0, 0.1)       |
-+-------------------------------------+
-|INTERVAL '0 00:00:00.1' DAY TO SECOND|
-+-------------------------------------+
-
-# overflow
-# >>> spark.sql("SELECT (make_dt_interval(214748364, 0, 0, 0))").show(truncate=False)
-# pyspark.errors.exceptions.captured.ArithmeticException: long overflow
-
-
-# doctest https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.make_dt_interval.html
->>> import pyspark.sql.functions as sf
->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec'])
->>> df.select('*', sf.make_dt_interval('day')).show(truncate=False)
-+---+----+---+--------+-----------------------------------+
-|day|hour|min|sec     |make_dt_interval(day, 0, 0, 0)     |
-+---+----+---+--------+-----------------------------------+
-|1  |12  |30 |1.001001|INTERVAL '1 00:00:00' DAY TO SECOND|
-+---+----+---+--------+-----------------------------------+
-
->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec'])
->>> df.select('*', sf.make_dt_interval(df.day, df.hour)).show(truncate=False)
-+---+----+---+--------+-----------------------------------+
-|day|hour|min|sec     |make_dt_interval(day, hour, 0, 0)  |
-+---+----+---+--------+-----------------------------------+
-|1  |12  |30 |1.001001|INTERVAL '1 12:00:00' DAY TO SECOND|
-+---+----+---+--------+-----------------------------------+
-
->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec'])
->>> df.select('*', sf.make_dt_interval(df.day, 'hour', df.min)).show(truncate=False)
-+---+----+---+--------+-----------------------------------+
-|day|hour|min|sec     |make_dt_interval(day, hour, min, 0)|
-+---+----+---+--------+-----------------------------------+
-|1  |12  |30 |1.001001|INTERVAL '1 12:30:00' DAY TO SECOND|
-+---+----+---+--------+-----------------------------------+
-
->>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec'])
->>> df.select('*', sf.make_dt_interval(df.day, df.hour, df.min, df.sec)).show(truncate=False)
-+---+----+---+--------+------------------------------------------+
-|day|hour|min|sec     |make_dt_interval(day, hour, min, sec)     |
-+---+----+---+--------+------------------------------------------+
-|1  |12  |30 |1.001001|INTERVAL '1 12:30:01.001001' DAY TO SECOND|
-+---+----+---+--------+------------------------------------------+
diff --git a/python/pysail/tests/spark/function/test_make_interval.txt b/python/pysail/tests/spark/function/test_make_interval.txt
deleted file mode 100644
index dcf59520c..000000000
--- a/python/pysail/tests/spark/function/test_make_interval.txt
+++ /dev/null
@@ -1,37 +0,0 @@
->>> from pyspark.sql import Row, functions as F
->>> data = [
-...     Row(y=1, m=2, w=3, d=4, h=5, mi=6, s=7.5),
-...     Row(y=None, m=2, w=3, d=4, h=5, mi=6, s=7.5),
-...     Row(y=1, m=None, w=3, d=4, h=5, mi=6, s=7.5),
-...     Row(y=1, m=2, w=None, d=4, h=5, mi=6, s=7.5),
-...     Row(y=1, m=2, w=3, d=None, h=5, mi=6, s=7.5),
-...     Row(y=1, m=2, w=3, d=4, h=None, mi=6, s=7.5),
-...     Row(y=1, m=2, w=3, d=4, h=5, mi=None, s=7.5),
-...     Row(y=1, m=2, w=3, d=4, h=5, mi=6, s=None),
-...     Row(y=1, m=1, w=1, d=1, h=1, mi=1, s=1.0),
-...     # NaN and infinite
-...     Row(y=0, m=0, w=0, d=0, h=0, mi=0, s=float('nan')),
-...     Row(y=0, m=0, w=0, d=0, h=0, mi=0, s=float('inf')),
-...     Row(y=0, m=0, w=0, d=0, h=0, mi=0, s=float('-inf')),
-... ]
->>> df = spark.createDataFrame(data)
->>> df.withColumn(
-...     "interval",
-...     F.make_interval("y", "m", "w", "d", "h", "mi", "s")
-... ).show(truncate=False)
-+----+----+----+----+----+----+---------+------------------------------------------------------+
-|y   |m   |w   |d   |h   |mi  |s        |interval                                              |
-+----+----+----+----+----+----+---------+------------------------------------------------------+
-|1   |2   |3   |4   |5   |6   |7.5      |1 years 2 months 25 days 5 hours 6 minutes 7.5 seconds|
-|NULL|2   |3   |4   |5   |6   |7.5      |NULL                                                  |
-|1   |NULL|3   |4   |5   |6   |7.5      |NULL                                                  |
-|1   |2   |NULL|4   |5   |6   |7.5      |NULL                                                  |
-|1   |2   |3   |NULL|5   |6   |7.5      |NULL                                                  |
-|1   |2   |3   |4   |NULL|6   |7.5      |NULL                                                  |
-|1   |2   |3   |4   |5   |NULL|7.5      |NULL                                                  |
-|1   |2   |3   |4   |5   |6   |NULL     |NULL                                                  |
-|1   |1   |1   |1   |1   |1   |1.0      |1 years 1 months 8 days 1 hours 1 minutes 1 seconds   |
-|0   |0   |0   |0   |0   |0   |NaN      |NULL                                                  |
-|0   |0   |0   |0   |0   |0   |Infinity |NULL                                                  |
-|0   |0   |0   |0   |0   |0   |-Infinity|NULL                                                  |
-+----+----+----+----+----+----+---------+------------------------------------------------------+
diff --git a/python/pysail/tests/spark/function/test_map_from_arrays.txt b/python/pysail/tests/spark/function/test_map_from_arrays.txt
deleted file mode 100644
index 3d4794681..000000000
--- a/python/pysail/tests/spark/function/test_map_from_arrays.txt
+++ /dev/null
@@ -1,30 +0,0 @@
->>> spark.sql("""
-...    select map_from_arrays(a, b) as result, typeof(map_from_arrays(a, b)) as type
-...    from values 
-...        (array(1), array('a')), 
-...        (NULL, NULL),
-...        (array(1,2,3), NULL),
-...        (NULL, array('b', 'c')), 
-...        (array(4, 5), array('d', 'e')), 
-...        (array(), array()),
-...        (array(6, 7, 8), array('f', 'g', 'h')) 
-...    as tab(a, b);
-... """).show(truncate=False)
-+------------------------+---------------+
-|result                  |type           |
-+------------------------+---------------+
-|{1 -> a}                |map|
-|NULL                    |map|
-|NULL                    |map|
-|NULL                    |map|
-|{4 -> d, 5 -> e}        |map|
-|{}                      |map|
-|{6 -> f, 7 -> g, 8 -> h}|map|
-+------------------------+---------------+
-
->>> spark.sql("select map_from_arrays(NULL, NULL) as result, typeof(map_from_arrays(NULL, NULL)) as type").show(truncate=False)
-+------+--------------+
-|result|type          |
-+------+--------------+
-|NULL  |map|
-+------+--------------+
diff --git a/python/pysail/tests/spark/function/test_map_from_entries.txt b/python/pysail/tests/spark/function/test_map_from_entries.txt
deleted file mode 100644
index c6519bfe9..000000000
--- a/python/pysail/tests/spark/function/test_map_from_entries.txt
+++ /dev/null
@@ -1,51 +0,0 @@
->>> spark.sql("""
-... SELECT map_from_entries(data) as result, typeof(map_from_entries(data)) as type
-... from VALUES
-...     (NULL), 
-...     (
-...         array(struct(1 as a, 'b' as b), 
-...         struct(2 as a, NULL as b), 
-...         struct(3 as a, 'd' as b))
-...     ), 
-...     (array()),
-...     (cast(NULL as array>)) 
-... as tab(data);
-... """).show(truncate=False)
-+---------------------------+---------------+
-|result                     |type           |
-+---------------------------+---------------+
-|NULL                       |map|
-|{1 -> b, 2 -> NULL, 3 -> d}|map|
-|{}                         |map|
-|NULL                       |map|
-+---------------------------+---------------+
-
->>> spark.sql("""
-... SELECT map_from_entries(data) as result, typeof(map_from_entries(data)) as type
-... from values
-...     (array(
-...         struct(1 as a, "a" as b), 
-...         cast(NULL as struct), 
-...         cast(NULL as struct)
-...     )),
-...     (NULL),
-...     (array(
-...         struct(2 as a, "b" as b), 
-...         struct(3 as a, "c" as b)
-...     )),
-...     (array(
-...         struct(4 as a, "d" as b),
-...         cast(NULL as struct),
-...         struct(5 as a, "e" as b),
-...         struct(6 as a, "f" as b)
-...     ))
-... as tab(data);
-... """).show(truncate=False)
-+----------------+---------------+
-|result          |type           |
-+----------------+---------------+
-|NULL            |map|
-|NULL            |map|
-|{2 -> b, 3 -> c}|map|
-|NULL            |map|
-+----------------+---------------+
\ No newline at end of file
diff --git a/python/pysail/tests/spark/function/test_nullability_schema.py b/python/pysail/tests/spark/function/test_nullability_schema.py
new file mode 100644
index 000000000..f214a2358
--- /dev/null
+++ b/python/pysail/tests/spark/function/test_nullability_schema.py
@@ -0,0 +1,141 @@
+"""Output-schema nullability via the DataFrame API.
+
+Complements the SQL `@function(nullability)` feature suite with the DataFrame path: build a
+DataFrame from an explicit `StructType` (so the input nullability is exactly what
+we declare — a non-nullable and a nullable column), apply functions through the
+`F.*` API, and assert the resulting schema's `nullable` flags.
+
+Spark's `nullable` flag is static and, for a deterministic function that only
+transforms its arguments, follows the inputs:
+  - a non-null column   -> non-nullable output
+  - a nullable column   -> nullable output
+
+Divergences (Sail declares `nullable` differently from Spark) are marked
+`xfail(strict=True)` off JVM, the same contract as the `@sail-bug` feature tag.
+
+NOTE — this file is a representative sample, not exhaustive. The DataFrame-API
+path and the SQL path are NOT equivalent: some divergences only show up via SQL
+constant-folding. For example `bround` on a decimal is tagged `@sail-bug` in the
+SQL suite (the folded `SELECT bround(2.5, 0)` path) yet matches Spark through the
+DataFrame API (both column and `F.lit`) — so it passes here. `round` diverges on
+both paths, which is why it stays a bug. A full DataFrame-API sweep across all
+functions is a worthwhile follow-up: it is the path real PySpark code takes, and
+it catches (and clears) different cases than the SQL sweep.
+"""
+
+from decimal import Decimal
+
+import pytest
+from pyspark.sql import functions as F  # noqa: N812
+from pyspark.sql import types as T  # noqa: N812
+
+from pysail.testing.spark.utils.common import is_jvm_spark
+
+sail_bug = pytest.mark.xfail(not is_jvm_spark(), reason="Known Sail nullability bug", strict=True)
+
+
+def _frame(spark):
+    """One non-nullable ("nn") and one nullable ("nl") string column."""
+    schema = T.StructType(
+        [
+            T.StructField("nn", T.StringType(), nullable=False),
+            T.StructField("nl", T.StringType(), nullable=True),
+        ]
+    )
+    return spark.createDataFrame([("hello", "world")], schema)
+
+
+def _decimal_frame(spark):
+    """One non-nullable ("nn") and one nullable ("nl") decimal column."""
+    schema = T.StructType(
+        [
+            T.StructField("nn", T.DecimalType(2, 1), nullable=False),
+            T.StructField("nl", T.DecimalType(2, 1), nullable=True),
+        ]
+    )
+    return spark.createDataFrame([(Decimal("2.5"), Decimal("2.5"))], schema)
+
+
+def _nullable(df, column):
+    return df.schema[column].nullable
+
+
+def test_input_schema_is_honored(spark):
+    df = _frame(spark)
+    assert _nullable(df, "nn") is False
+    assert _nullable(df, "nl") is True
+
+
+# --- functions that correctly propagate nullability (pass on Sail and JVM) ---
+
+
+def test_length_propagates_nullability(spark):
+    out = _frame(spark).select(
+        F.length("nn").alias("from_nn"),
+        F.length("nl").alias("from_nl"),
+    )
+    assert _nullable(out, "from_nn") is False
+    assert _nullable(out, "from_nl") is True
+
+
+def test_concat_propagates_nullability(spark):
+    out = _frame(spark).select(
+        F.concat("nn", "nn").alias("both_non_null"),
+        F.concat("nn", "nl").alias("one_nullable"),
+    )
+    assert _nullable(out, "both_non_null") is False
+    assert _nullable(out, "one_nullable") is True
+
+
+def test_coalesce_propagates_nullability(spark):
+    out = _frame(spark).select(
+        F.coalesce("nn", "nl").alias("has_non_null"),
+        F.coalesce("nl", "nl").alias("all_nullable"),
+    )
+    assert _nullable(out, "has_non_null") is False
+    assert _nullable(out, "all_nullable") is True
+
+
+# --- functions where Sail is over-nullable ---
+# Both cases are checked so we see exactly where each is right and wrong:
+# Sail gets the nullable input right (nullable -> nullable) but the non-null
+# input wrong (non-null -> Sail still says nullable, Spark says non-nullable).
+
+
+@pytest.mark.parametrize("fn", [F.upper, F.lower, F.ascii, F.initcap])
+def test_pure_string_fn_of_nullable_is_nullable(spark, fn):
+    # nullable input -> nullable output: Sail and Spark agree
+    out = _frame(spark).select(fn("nl").alias("result"))
+    assert _nullable(out, "result") is True
+
+
+@pytest.mark.parametrize("fn", [F.upper, F.lower, F.ascii, F.initcap])
+@sail_bug
+def test_pure_string_fn_of_non_null_is_non_nullable(spark, fn):
+    # non-null input -> non-nullable output: Sail diverges (says nullable=True)
+    out = _frame(spark).select(fn("nn").alias("result"))
+    assert _nullable(out, "result") is False
+
+
+# --- the dangerous direction: Sail UNDER-nullable ---
+# round on a decimal can overflow, so Spark declares it nullable=True even for a
+# non-null input. Sail declares it non-nullable -> if a NULL ever appears an
+# optimizer could drop null-checks, so this is a correctness risk, not just
+# schema fidelity. (bround matches Spark here, kept as the passing counterpart.)
+
+
+def test_round_of_nullable_decimal_is_nullable(spark):
+    out = _decimal_frame(spark).select(F.round("nl", 1).alias("result"))
+    assert _nullable(out, "result") is True  # Sail and Spark agree
+
+
+def test_bround_of_non_null_decimal_is_nullable(spark):
+    out = _decimal_frame(spark).select(F.bround("nn", 1).alias("result"))
+    assert _nullable(out, "result") is True  # Sail and Spark agree
+
+
+@sail_bug
+def test_round_of_non_null_decimal_stays_nullable(spark):
+    # Spark keeps round(decimal) nullable (overflow); Sail says non-nullable.
+    out = _decimal_frame(spark).select(F.round("nn", 1).alias("result"))
+    assert _nullable(out, "result") is True
diff --git a/python/pysail/tests/spark/function/test_nullifzero.txt b/python/pysail/tests/spark/function/test_nullifzero.txt
deleted file mode 100644
index 5dd661f5c..000000000
--- a/python/pysail/tests/spark/function/test_nullifzero.txt
+++ /dev/null
@@ -1,97 +0,0 @@
->>> spark.sql("SELECT nullifzero(0), typeof(nullifzero(0))").show()
-+-------------+---------------------+
-|nullifzero(0)|typeof(nullifzero(0))|
-+-------------+---------------------+
-|         NULL|                  int|
-+-------------+---------------------+
-
->>> spark.sql("SELECT nullifzero(2), typeof(nullifzero(2))").show()
-+-------------+---------------------+
-|nullifzero(2)|typeof(nullifzero(2))|
-+-------------+---------------------+
-|            2|                  int|
-+-------------+---------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(0 AS TINYINT)), typeof(nullifzero(CAST(0 AS TINYINT)))").show()
-+------------------------------+--------------------------------------+
-|nullifzero(CAST(0 AS TINYINT))|typeof(nullifzero(CAST(0 AS TINYINT)))|
-+------------------------------+--------------------------------------+
-|                          NULL|                               tinyint|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(2 AS TINYINT)), typeof(nullifzero(CAST(2 AS TINYINT)))").show()
-+------------------------------+--------------------------------------+
-|nullifzero(CAST(2 AS TINYINT))|typeof(nullifzero(CAST(2 AS TINYINT)))|
-+------------------------------+--------------------------------------+
-|                             2|                               tinyint|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(0 AS SMALLINT)), typeof(nullifzero(CAST(0 AS SMALLINT)))").show()
-+-------------------------------+---------------------------------------+
-|nullifzero(CAST(0 AS SMALLINT))|typeof(nullifzero(CAST(0 AS SMALLINT)))|
-+-------------------------------+---------------------------------------+
-|                           NULL|                               smallint|
-+-------------------------------+---------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(0 AS BIGINT)), typeof(nullifzero(CAST(0 AS BIGINT)))").show()
-+-----------------------------+-------------------------------------+
-|nullifzero(CAST(0 AS BIGINT))|typeof(nullifzero(CAST(0 AS BIGINT)))|
-+-----------------------------+-------------------------------------+
-|                         NULL|                               bigint|
-+-----------------------------+-------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(0.0 AS FLOAT)), typeof(nullifzero(CAST(0.0 AS FLOAT)))").show()
-+------------------------------+--------------------------------------+
-|nullifzero(CAST(0.0 AS FLOAT))|typeof(nullifzero(CAST(0.0 AS FLOAT)))|
-+------------------------------+--------------------------------------+
-|                          NULL|                                 float|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(1.5 AS FLOAT)), typeof(nullifzero(CAST(1.5 AS FLOAT)))").show()
-+------------------------------+--------------------------------------+
-|nullifzero(CAST(1.5 AS FLOAT))|typeof(nullifzero(CAST(1.5 AS FLOAT)))|
-+------------------------------+--------------------------------------+
-|                           1.5|                                 float|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(0.0 AS DOUBLE)), typeof(nullifzero(CAST(0.0 AS DOUBLE)))").show()
-+-------------------------------+---------------------------------------+
-|nullifzero(CAST(0.0 AS DOUBLE))|typeof(nullifzero(CAST(0.0 AS DOUBLE)))|
-+-------------------------------+---------------------------------------+
-|                           NULL|                                 double|
-+-------------------------------+---------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(2.5 AS DOUBLE)), typeof(nullifzero(CAST(2.5 AS DOUBLE)))").show()
-+-------------------------------+---------------------------------------+
-|nullifzero(CAST(2.5 AS DOUBLE))|typeof(nullifzero(CAST(2.5 AS DOUBLE)))|
-+-------------------------------+---------------------------------------+
-|                            2.5|                                 double|
-+-------------------------------+---------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(0.00 AS DECIMAL(10,2))), typeof(nullifzero(CAST(0.00 AS DECIMAL(10,2))))").show()
-+---------------------------------------+-----------------------------------------------+
-|nullifzero(CAST(0.00 AS DECIMAL(10,2)))|typeof(nullifzero(CAST(0.00 AS DECIMAL(10,2))))|
-+---------------------------------------+-----------------------------------------------+
-|                                   NULL|                                  decimal(10,2)|
-+---------------------------------------+-----------------------------------------------+
-
->>> spark.sql("SELECT nullifzero(CAST(5.25 AS DECIMAL(10,2))), typeof(nullifzero(CAST(5.25 AS DECIMAL(10,2))))").show()
-+---------------------------------------+-----------------------------------------------+
-|nullifzero(CAST(5.25 AS DECIMAL(10,2)))|typeof(nullifzero(CAST(5.25 AS DECIMAL(10,2))))|
-+---------------------------------------+-----------------------------------------------+
-|                                   5.25|                                  decimal(10,2)|
-+---------------------------------------+-----------------------------------------------+
-
->>> spark.sql("SELECT nullifzero(-1), typeof(nullifzero(-1))").show()
-+-----------------+-------------------------+
-|nullifzero((- 1))|typeof(nullifzero((- 1)))|
-+-----------------+-------------------------+
-|               -1|                      int|
-+-----------------+-------------------------+
-
->>> spark.sql("SELECT nullifzero(NULL), typeof(nullifzero(NULL))").show()
-+----------------+------------------------+
-|nullifzero(NULL)|typeof(nullifzero(NULL))|
-+----------------+------------------------+
-|            NULL|                     int|
-+----------------+------------------------+
diff --git a/python/pysail/tests/spark/function/test_parse_url.txt b/python/pysail/tests/spark/function/test_parse_url.txt
deleted file mode 100644
index 68794d619..000000000
--- a/python/pysail/tests/spark/function/test_parse_url.txt
+++ /dev/null
@@ -1,98 +0,0 @@
->>> spark.sql("SELECT parse_url('https://example.com/a?x=1', 'QUERY', 'x') AS result, typeof(parse_url('https://example.com/a?x=1', 'QUERY', 'x')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|1     |string|
-+------+------+
-
->>> spark.sql("SELECT parse_url('https://example.com/path#frag', 'REF') AS result, typeof(parse_url('https://example.com/path#frag', 'REF')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|frag  |string|
-+------+------+
-
->>> spark.sql("SELECT parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO') AS result, typeof(parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO')) AS type").show(truncate=False)
-+--------+------+
-|result  |type  |
-+--------+------+
-|user:pwd|string|
-+--------+------+
-
->>> spark.sql("SELECT parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST') AS result, typeof(parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST')) AS type").show(truncate=False)
-+-------------+------+
-|result       |type  |
-+-------------+------+
-|[2001:db8::2]|string|
-+-------------+------+
-
->>> spark.sql("SELECT parse_url('https://example.com', 'PATH') AS result, typeof(parse_url('https://example.com', 'PATH')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|      |string|
-+------+------+
-
->>> spark.sql("SELECT parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL') AS result, typeof(parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|https |string|
-+------+------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'HOST')").show(truncate=False)
-+------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, HOST)|
-+------------------------------------------------------------------+
-|spark.apache.org                                                  |
-+------------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'PATH')").show(truncate=False)
-+------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, PATH)|
-+------------------------------------------------------------------+
-|/path                                                             |
-+------------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'QUERY')").show(truncate=False)
-+-------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, QUERY)|
-+-------------------------------------------------------------------+
-|query=1                                                            |
-+-------------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'REF')").show(truncate=False)
-+-----------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, REF)|
-+-----------------------------------------------------------------+
-|Ref                                                              |
-+-----------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'PROTOCOL')").show(truncate=False)
-+----------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, PROTOCOL)|
-+----------------------------------------------------------------------+
-|http                                                                  |
-+----------------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'FILE')").show(truncate=False)
-+------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, FILE)|
-+------------------------------------------------------------------+
-|/path?query=1                                                     |
-+------------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'AUTHORITY')").show(truncate=False)
-+-----------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, AUTHORITY)|
-+-----------------------------------------------------------------------+
-|userinfo@spark.apache.org                                              |
-+-----------------------------------------------------------------------+
-
->>> spark.sql("select parse_url('http://userinfo@spark.apache.org/path?query=1#Ref', 'USERINFO')").show(truncate=False)
-+----------------------------------------------------------------------+
-|parse_url(http://userinfo@spark.apache.org/path?query=1#Ref, USERINFO)|
-+----------------------------------------------------------------------+
-|userinfo                                                              |
-+----------------------------------------------------------------------+
-
diff --git a/python/pysail/tests/spark/function/test_percentile_disc.py b/python/pysail/tests/spark/function/test_percentile_disc.py
index 5a783dbad..4327edc8e 100644
--- a/python/pysail/tests/spark/function/test_percentile_disc.py
+++ b/python/pysail/tests/spark/function/test_percentile_disc.py
@@ -16,7 +16,8 @@ def test_percentile_disc_basic(spark):
 
     # Sorted: [10, 20, 30, 40, 50]
     # 50th percentile (discrete) = 30 (actual value, not interpolated)
-    expected = pd.DataFrame({"median": [30]})
+    # Spark always returns DOUBLE from percentile_disc, regardless of input type.
+    expected = pd.DataFrame({"median": [30.0]})
     pd.testing.assert_frame_equal(actual, expected)
 
 
@@ -47,7 +48,7 @@ def test_percentile_disc_with_group_by(spark):
 
     # Group A: [10, 20, 30] -> median = 20
     # Group B: [100, 200, 300, 400] -> median = 200
-    expected = pd.DataFrame({"category": ["A", "B"], "median": [20, 200]})
+    expected = pd.DataFrame({"category": ["A", "B"], "median": [20.0, 200.0]})
     pd.testing.assert_frame_equal(actual, expected)
 
 
@@ -68,7 +69,7 @@ def test_percentile_disc_edge_cases(spark):
 
     # 0th percentile = min = 5
     # 100th percentile = max = 20
-    expected = pd.DataFrame({"p0": [5], "p100": [20]})
+    expected = pd.DataFrame({"p0": [5.0], "p100": [20.0]})
     pd.testing.assert_frame_equal(actual, expected)
 
 
@@ -87,5 +88,5 @@ def test_percentile_disc_with_nulls(spark):
 
     # Non-null values: [10, 20, 30, 40, 50]
     # 50th percentile (discrete) = 30
-    expected = pd.DataFrame({"median": [30]})
+    expected = pd.DataFrame({"median": [30.0]})
     pd.testing.assert_frame_equal(actual, expected)
diff --git a/python/pysail/tests/spark/function/test_schema_of_json.txt b/python/pysail/tests/spark/function/test_schema_of_json.txt
deleted file mode 100644
index 295d6892b..000000000
--- a/python/pysail/tests/spark/function/test_schema_of_json.txt
+++ /dev/null
@@ -1,323 +0,0 @@
->>> from pyspark.sql.functions import lit, schema_of_json
-
->>> df = spark.range(1)
->>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect()
-[Row(json='STRUCT')]
-
-The `allowUnquotedFieldNames` option allows parsing JSON strings with
-unquoted field names.
-
->>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames': 'true'})
->>> df.select(schema.alias("json")).collect()
-[Row(json='STRUCT')]
-
-Option keys and boolean option values are case-insensitive.
-
->>> schema = schema_of_json('{a: 1}', {'ALLOWUNQUOTEDFIELDNAMES': 'True'})
->>> df.select(schema.alias("json")).collect()
-[Row(json='STRUCT')]
-
-Quoted field names and string values are left untouched, including in
-nested structures.
-
->>> schema = schema_of_json(
-...     '{a: {"b": [1.5], c: "x: {y: 1}"}}',
-...     {'allowUnquotedFieldNames': 'true'},
-... )
->>> df.select(schema.alias("json")).collect()
-[Row(json='STRUCT, c: STRING>>')]
-
-The options are excluded from the default column name, which only contains
-the JSON string argument.
-
->>> df.select(schema_of_json('{a: 1}', {'allowUnquotedFieldNames': 'true'})).columns
-['schema_of_json({a: 1})']
-
-The option can be explicitly disabled.
-
->>> schema = schema_of_json('{"a": 2}', {'allowUnquotedFieldNames': 'false'})
->>> df.select(schema.alias("json")).collect()
-[Row(json='STRUCT')]
-
-Invalid boolean option values are rejected. Both Sail and JVM Spark raise an
-error here, but with different exception classes (Sail raises an
-`AnalysisException`, while JVM Spark fails to parse the value with Scala's
-`.toBoolean` and raises an `IllegalArgumentException`). Since a doctest
-cannot match multiple exception classes, we catch the error explicitly so
-that the test passes against both engines.
-
->>> schema = schema_of_json('{"a": 1}', {'allowUnquotedFieldNames': 'yes'})
->>> try:
-...     df.select(schema.alias("json")).collect()
-... except Exception:
-...     print("error")
-error
-
-All expected values below were captured from JVM Spark (verified identical on
-3.5.7 and 4.1.1).
-
->>> def check(json_str, opts=None):
-...     col = schema_of_json(json_str) if opts is None else schema_of_json(json_str, opts)
-...     return df.select(col.alias("r")).collect()[0]["r"]
-
-Unknown options are silently ignored, matching Spark.
-
->>> check('{"a": 1}', {'myImaginaryOption': 'banana'})
-'STRUCT'
-
-Single quotes are allowed by default (`allowSingleQuotes` defaults to true in
-Spark) and can be disabled explicitly.
-
->>> check('{"a": \'x\'}')
-'STRUCT'
->>> check("{'a': 1}")
-'STRUCT'
->>> check(r"{'a': 'it\'s'}")
-'STRUCT'
->>> check('{\'a\': \'say "hi"\'}')
-'STRUCT'
->>> check('{"a": "it\'s"}')
-'STRUCT'
->>> try:
-...     check("{'a': 1}", {'allowSingleQuotes': 'false'})
-... except Exception:
-...     print("error")
-error
-
-Field names that are not valid identifiers are quoted with backticks in the
-DDL output, with embedded backticks doubled.
-
->>> check('{"x y": 1}')
-'STRUCT<`x y`: BIGINT>'
->>> check('{"x-y": 1}')
-'STRUCT<`x-y`: BIGINT>'
->>> check('{"x.y": 1}')
-'STRUCT<`x.y`: BIGINT>'
->>> check('{"1a": 1}')
-'STRUCT<`1a`: BIGINT>'
->>> check('{"_a": 1}')
-'STRUCT<_a: BIGINT>'
->>> check('{"ab1_": 1}')
-'STRUCT'
->>> check('{"x`y": 1}')
-'STRUCT<`x``y`: BIGINT>'
->>> check('{"café": 1}')
-'STRUCT<`café`: BIGINT>'
->>> check('{"": 1}')
-'STRUCT<>'
-
-Booleans and nulls.
-
->>> check('{"a": true}')
-'STRUCT'
->>> check('{"a": false}')
-'STRUCT'
->>> check('{"a": null}')
-'STRUCT'
->>> check('null')
-'STRING'
->>> check('{"a": {"b": null}}')
-'STRUCT>'
->>> check('{"a": [null]}')
-'STRUCT>'
->>> check('{"a": [null, 1]}')
-'STRUCT>'
-
-Array element types are merged to the most specific common type.
-
->>> check('{"a": [1, "x"]}')
-'STRUCT>'
->>> check('{"a": [1, 2.5]}')
-'STRUCT>'
->>> check('{"a": [1, 2.5, "x"]}')
-'STRUCT>'
->>> check('{"a": [1, true]}')
-'STRUCT>'
->>> check('{"a": [{"b": 1}, {"c": 2}]}')
-'STRUCT>>'
->>> check('{"a": [{"b": 1}, {"b": "x"}]}')
-'STRUCT>>'
->>> check('{"a": [{"b": 1}, 2]}')
-'STRUCT>'
->>> check('{"a": [[1], [2.5]]}')
-'STRUCT>>'
-
-Empty structs are dropped during canonicalization.
-
->>> check('{"a": {}}')
-'STRUCT<>'
->>> check('{}')
-'STRUCT<>'
-
-Numbers: integers that fit in a long are BIGINT, wider integral values are
-DECIMAL(p,0) up to the maximum precision of 38, and everything else is
-DOUBLE. `prefersDecimal` infers floating-point values as decimals.
-
->>> check('{"a": 9223372036854775807}')
-'STRUCT'
->>> check('{"a": 9223372036854775808}')
-'STRUCT'
->>> check('{"a": 99999999999999999999999999}')
-'STRUCT'
->>> check('{"a": -99999999999999999999999999}')
-'STRUCT'
->>> check('{"a": ' + '9' * 39 + '}')
-'STRUCT'
->>> check('{"a": 1.7976931348623157E308}')
-'STRUCT'
->>> check('{"a": 1E309}')
-'STRUCT'
->>> check('{"a": 12.345678901234567890123}')
-'STRUCT'
->>> check('{"a": 1.5}', {'prefersDecimal': 'true'})
-'STRUCT'
->>> check('{"a": 99999999999999999999999999}', {'prefersDecimal': 'true'})
-'STRUCT'
-
-With `prefersDecimal`, a value with an exponent follows Java `BigDecimal`
-semantics: the scale is the number of fraction digits minus the exponent,
-and the precision counts the unscaled digits, floored at the scale. A
-negative scale is an error on both engines.
-
->>> PD = {'prefersDecimal': 'true'}
->>> check('{"a": 1.55E1}', PD)
-'STRUCT'
->>> check('{"a": 1.5E-2}', PD)
-'STRUCT'
->>> check('{"a": 1E-2}', PD)
-'STRUCT'
->>> check('{"a": 15E-1}', PD)
-'STRUCT'
->>> check('{"a": 1.500E1}', PD)
-'STRUCT'
->>> check('{"a": 1.5E0}', PD)
-'STRUCT'
->>> check('{"a": 1.5E2}', PD)
-Traceback (most recent call last):
-...
-pyspark.errors.exceptions.connect.AnalysisException: Negative scale is not allowed: '-1'
->>> check('{"a": 1E2}', PD)
-Traceback (most recent call last):
-...
-pyspark.errors.exceptions.connect.AnalysisException: Negative scale is not allowed: '-2'
->>> check('{"a": 15E2}', PD)
-Traceback (most recent call last):
-...
-pyspark.errors.exceptions.connect.AnalysisException: Negative scale is not allowed: '-2'
->>> check('{"a": 1.5E2}')
-'STRUCT'
-
-Non-numeric numbers are allowed by default (`allowNonNumericNumbers`
-defaults to true in Spark).
-
->>> check('{"a": NaN}')
-'STRUCT'
->>> check('{"a": Infinity}')
-'STRUCT'
-
-`allowNumericLeadingZeros` allows leading zeros in integral numbers, which
-are rejected by default. Leading zeros do not count toward the decimal
-precision.
-
->>> LZ = {'allowNumericLeadingZeros': 'true'}
->>> check('{"a": 01}', LZ)
-'STRUCT'
->>> check('{"a": 0001}', LZ)
-'STRUCT'
->>> check('{"a": -01}', LZ)
-'STRUCT'
->>> check('{"a": 007}', LZ)
-'STRUCT'
->>> check('{"a": 0}', LZ)
-'STRUCT'
->>> check('{"a": 00}', LZ)
-'STRUCT'
->>> check('{"a": -00}', LZ)
-'STRUCT'
->>> check('{"a": 01.5}', LZ)
-'STRUCT'
->>> check('{"a": 00.5}', LZ)
-'STRUCT'
->>> check('{"a": 01e2}', LZ)
-'STRUCT'
->>> check('{"a": 009223372036854775808}', LZ)
-'STRUCT'
->>> check('{"a": 00' + '9' * 38 + '}', LZ)
-'STRUCT'
->>> check('{"a": 00' + '9' * 39 + '}', LZ)
-'STRUCT'
->>> check('{"01b": 01, "b": "01"}', LZ)
-'STRUCT<`01b`: BIGINT, b: STRING>'
-
-Leading zeros in the exponent part are always valid, regardless of the
-option.
-
->>> check('{"a": 1e02}', LZ)
-'STRUCT'
->>> check('{"a": 1e02}')
-'STRUCT'
-
-`allowNumericLeadingZeros` combines with `prefersDecimal`.
-
->>> check('{"a": 00.50}', {'allowNumericLeadingZeros': 'true', 'prefersDecimal': 'true'})
-'STRUCT'
->>> check('{"a": 01.5}', {'allowNumericLeadingZeros': 'true', 'prefersDecimal': 'true'})
-'STRUCT'
-
-Leading zeros are rejected by default and when the option is explicitly
-disabled. Both engines raise an error, with different exception classes.
-
->>> try:
-...     check('{"a": 01}')
-... except Exception:
-...     print("error")
-error
->>> try:
-...     check('{"a": 01}', {'allowNumericLeadingZeros': 'false'})
-... except Exception:
-...     print("error")
-error
-
-The unquoted field name character set matches Jackson: ASCII alphanumeric
-characters, `_$@#*+-`, and all non-ASCII characters. A `.` is not a valid
-name character.
-
->>> UQ = {'allowUnquotedFieldNames': 'true'}
->>> check('{+a: 1}', UQ)
-'STRUCT<`+a`: BIGINT>'
->>> check('{a@b: 1}', UQ)
-'STRUCT<`a@b`: BIGINT>'
->>> check('{a#b: 1}', UQ)
-'STRUCT<`a#b`: BIGINT>'
->>> check('{a*b: 1}', UQ)
-'STRUCT<`a*b`: BIGINT>'
->>> check('{a$b: 1}', UQ)
-'STRUCT<`a$b`: BIGINT>'
->>> check('{$a: 1}', UQ)
-'STRUCT<`$a`: BIGINT>'
->>> check('{_a: 1}', UQ)
-'STRUCT<_a: BIGINT>'
->>> check('{9a: 1}', UQ)
-'STRUCT<`9a`: BIGINT>'
->>> check('{a9: 1}', UQ)
-'STRUCT'
->>> check('{ключ: 1}', UQ)
-'STRUCT<`ключ`: BIGINT>'
->>> check('{aé: 1}', UQ)
-'STRUCT<`aé`: BIGINT>'
->>> try:
-...     check('{.a: 1}', UQ)
-... except Exception:
-...     print("error")
-error
->>> for s in ['{`a: 1}', '{a`b: 1}', '{a: 1}', '{\\a: 1}']:
-...     try:
-...         check(s, UQ)
-...     except Exception:
-...         print("error")
-error
-error
-error
-error
-error
-error
diff --git a/python/pysail/tests/spark/function/test_sentences.txt b/python/pysail/tests/spark/function/test_sentences.txt
deleted file mode 100644
index ce534cf98..000000000
--- a/python/pysail/tests/spark/function/test_sentences.txt
+++ /dev/null
@@ -1,55 +0,0 @@
->>> spark.sql("SELECT sentences('Hi there! Good morning.') AS result").show(truncate=False)
-+...
-|result...|
-+...
-|[[Hi, there], [Good, morning]]...|
-+...
-
-
->>> spark.sql("SELECT sentences('Hello world') AS result").show(truncate=False)
-+...
-|result...|
-+...
-|[[Hello, world]]...|
-+...
-
-
->>> spark.sql("SELECT sentences('What? Yes! OK.') AS result").show(truncate=False)
-+...
-|result...|
-+...
-|[[What], [Yes], [OK]]...|
-+...
-
-
->>> spark.sql("SELECT sentences(NULL) AS result").show(truncate=False)
-+------+
-|result|
-+------+
-|NULL  |
-+------+
-
-
->>> spark.sql("SELECT sentences('') AS result").show(truncate=False)
-+------+
-|result|
-+------+
-|[]    |
-+------+
-
-
->>> spark.sql("SELECT sentences('Hi there!', 'en') AS result").show(truncate=False)
-+...
-|result...|
-+...
-|[[Hi, there]]...|
-+...
-
-
->>> spark.sql("SELECT sentences('Hi there!', 'en', 'US') AS result").show(truncate=False)
-+...
-|result...|
-+...
-|[[Hi, there]]...|
-+...
-
diff --git a/python/pysail/tests/spark/function/test_split.txt b/python/pysail/tests/spark/function/test_split.txt
deleted file mode 100644
index d64566839..000000000
--- a/python/pysail/tests/spark/function/test_split.txt
+++ /dev/null
@@ -1,129 +0,0 @@
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]') AS result, typeof(split('oneAtwoBthreeC', '[ABC]')) AS type").show(truncate=False)
-+-------------------+-------------+
-|result             |type         |
-+-------------------+-------------+
-|[one, two, three, ]|array|
-+-------------------+-------------+
-
->>> spark.sql("SELECT split('1A2B3C', '[1-9]+') AS result, typeof(split('1A2B3C', '[1-9]+')) AS type").show(truncate=False)
-+-----------+-------------+
-|result     |type         |
-+-----------+-------------+
-|[, A, B, C]|array|
-+-----------+-------------+
-
->>> spark.sql("SELECT split('aa2bb3cc4', NULL) AS result, typeof(split('aa2bb3cc4', NULL)) AS type").show(truncate=False)
-+------+-------------+
-|result|type         |
-+------+-------------+
-|NULL  |array|
-+------+-------------+
-
->>> spark.sql("SELECT split(NULL, '[1-9]+') AS result, typeof(split(NULL, '[1-9]+')) AS type").show(truncate=False)
-+------+-------------+
-|result|type         |
-+------+-------------+
-|NULL  |array|
-+------+-------------+
-
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', -4) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', -4)) AS type").show(truncate=False)
-+-------------------+-------------+
-|result             |type         |
-+-------------------+-------------+
-|[one, two, three, ]|array|
-+-------------------+-------------+
-
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', -1) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', -1)) AS type").show(truncate=False)
-+-------------------+-------------+
-|result             |type         |
-+-------------------+-------------+
-|[one, two, three, ]|array|
-+-------------------+-------------+
-
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 0) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 0)) AS type").show(truncate=False)
-+-------------------+-------------+
-|result             |type         |
-+-------------------+-------------+
-|[one, two, three, ]|array|
-+-------------------+-------------+
-
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 1) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 1)) AS type").show(truncate=False)
-+----------------+-------------+
-|result          |type         |
-+----------------+-------------+
-|[oneAtwoBthreeC]|array|
-+----------------+-------------+
-
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 2) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 2)) AS type").show(truncate=False)
-+-----------------+-------------+
-|result           |type         |
-+-----------------+-------------+
-|[one, twoBthreeC]|array|
-+-----------------+-------------+
-
->>> spark.sql("SELECT split('oneAtwoBthreeC', '[ABC]', 10) AS result, typeof(split('oneAtwoBthreeC', '[ABC]', 10)) AS type").show(truncate=False)
-+-------------------+-------------+
-|result             |type         |
-+-------------------+-------------+
-|[one, two, three, ]|array|
-+-------------------+-------------+
-
->>> spark.sql("SELECT split('1A2B3C', '[1-9]+', 1) AS result, typeof(split('1A2B3C', '[1-9]+', 1)) AS type").show(truncate=False)
-+--------+-------------+
-|result  |type         |
-+--------+-------------+
-|[1A2B3C]|array|
-+--------+-------------+
-
->>> spark.sql("SELECT split('aa2bb3cc4', '[1-9]+', -1) AS result, typeof(split('aa2bb3cc4', '[1-9]+', -1)) AS type").show(truncate=False)
-+--------------+-------------+
-|result        |type         |
-+--------------+-------------+
-|[aa, bb, cc, ]|array|
-+--------------+-------------+
-
->>> spark.sql("SELECT split('aa2bb3cc4', '[1-9]+', 2) AS result, typeof(split('aa2bb3cc4', '[1-9]+', 2)) AS type").show(truncate=False)
-+------------+-------------+
-|result      |type         |
-+------------+-------------+
-|[aa, bb3cc4]|array|
-+------------+-------------+
-
->>> spark.sql("SELECT split('aa2bb3cc4', '[1-9]+', NULL) AS result, typeof(split('aa2bb3cc4', '[1-9]+', NULL)) AS type").show(truncate=False)
-+------+-------------+
-|result|type         |
-+------+-------------+
-|NULL  |array|
-+------+-------------+
-
->>> spark.sql("SELECT split('aa2bb3cc4', NULL, -1) AS result, typeof(split('aa2bb3cc4', NULL, -1)) AS type").show(truncate=False)
-+------+-------------+
-|result|type         |
-+------+-------------+
-|NULL  |array|
-+------+-------------+
-
->>> spark.sql("SELECT split(NULL, '[1-9]+', -1) AS result, typeof(split(NULL, '[1-9]+', -1)) AS type").show(truncate=False)
-+------+-------------+
-|result|type         |
-+------+-------------+
-|NULL  |array|
-+------+-------------+
-
->>> spark.sql("SELECT split(s, p, l) AS result, typeof(split(s, p, l)) AS type FROM VALUES ('oneAtwoBthreeC', '[ABC]', -4), ('oneAtwoBthreeC', '[ABC]', -1), ('oneAtwoBthreeC', '[ABC]', 0), ('oneAtwoBthreeC', '[ABC]', 1), ('oneAtwoBthreeC', '[ABC]', 2), ('oneAtwoBthreeC', '[ABC]', 10), ('1A2B3C', '[1-9]+', 1), ('aa2bb3cc4', '[1-9]+', -1), ('aa2bb3cc4', '[1-9]+', 2), ('aa2bb3cc4', '[1-9]+', NULL), ('aa2bb3cc4', NULL, -1), (NULL, '[1-9]+', -1) AS T(s, p, l)").show(truncate=False)
-+-------------------+-------------+
-|result             |type         |
-+-------------------+-------------+
-|[one, two, three, ]|array|
-|[one, two, three, ]|array|
-|[one, two, three, ]|array|
-|[oneAtwoBthreeC]   |array|
-|[one, twoBthreeC]  |array|
-|[one, two, three, ]|array|
-|[1A2B3C]           |array|
-|[aa, bb, cc, ]     |array|
-|[aa, bb3cc4]       |array|
-|NULL               |array|
-|NULL               |array|
-|NULL               |array|
-+-------------------+-------------+
diff --git a/python/pysail/tests/spark/function/test_to_char.txt b/python/pysail/tests/spark/function/test_to_char.txt
deleted file mode 100644
index 4e17f5fe6..000000000
--- a/python/pysail/tests/spark/function/test_to_char.txt
+++ /dev/null
@@ -1,388 +0,0 @@
-Tests for `to_char`.
-
-All expected values below were captured from JVM Spark. Numeric cases are
-verified identical on 3.5.7 and 4.1.1. Datetime and binary inputs are only
-supported by JVM Spark 4.x (3.5 requires a DECIMAL input), so those examples
-are marked SAIL_ONLY; their expected values were captured from JVM Spark
-4.1.1.
-
->>> from pyspark.sql.functions import lit, to_char
-
->>> df = spark.createDataFrame([(78.12,)], ["e"])
->>> df.select(to_char(df.e, lit("$99.99")).alias("r")).collect()
-[Row(r='$78.12')]
-
-The helpers below evaluate a SQL expression and return the result value, or
-print "error" if the query fails. Error cases use the helper (instead of
-matching the exception text) because Sail and the two JVM Spark versions
-raise errors with different classes and messages.
-
->>> def c(expr):
-...     return spark.sql(f"SELECT {expr} AS r").collect()[0]["r"]
->>> def err(expr):
-...     try:
-...         spark.sql(f"SELECT {expr} AS r").collect()
-...         print("no error")
-...     except Exception:
-...         print("error")
-
-Digit sequences: '9' pads with spaces, '0' pads with zeros. A sequence
-starting with '0' before the decimal point zero-pads the whole sequence.
-
->>> c("to_char(454, '999')")
-'454'
->>> c("to_char(454, '9999')")
-' 454'
->>> c("to_char(454, '99999')")
-'  454'
->>> c("to_char(454, '0999')")
-'0454'
->>> c("to_char(454, '00999')")
-'00454'
->>> c("to_char(454, '0000')")
-'0454'
->>> c("to_char(454, '90999')")
-'  454'
->>> c("to_char(454, '99099')")
-'  454'
->>> c("to_char(1, '9')")
-'1'
->>> c("to_char(1, '0')")
-'1'
-
-Values with more digits than the format are replaced by '#' characters.
-
->>> c("to_char(454, '99')")
-'##'
->>> c("to_char(454, '00')")
-'##'
->>> c("to_char(123.456, '999D9')")
-'###.#'
->>> c("to_char(78.123, '99.9')")
-'##.#'
->>> c("to_char(12345, '9,999')")
-'# ###'
->>> c("to_char(0.05, '9.9')")
-'#.#'
-
-Zero values.
-
->>> c("to_char(0, '9')")
-' '
->>> c("to_char(0, '99')")
-'  '
->>> c("to_char(0, '0')")
-'0'
->>> c("to_char(0, '00')")
-'00'
->>> c("to_char(0, '99.99')")
-' 0.00'
->>> c("to_char(0, '00.00')")
-'00.00'
->>> c("to_char(0, 'S99')")
-'  +'
->>> c("to_char(0, '99S')")
-'+  '
->>> c("to_char(0, '$99.99')")
-'$ 0.00'
->>> c("to_char(0.0, '9.9')")
-'0.0'
-
-Decimal point ('.' or 'D').
-
->>> c("to_char(78, '99.99')")
-'78.00'
->>> c("to_char(78, '99.')")
-'78 '
->>> c("to_char(78, '99D')")
-'78 '
->>> c("to_char(0.45, '.99')")
-'.45'
->>> c("to_char(0.45, 'D99')")
-'.45'
->>> c("to_char(0.45, '9.99')")
-'0.45'
->>> c("to_char(0.45, '99.99')")
-' 0.45'
-
-Grouping separators (',' or 'G').
-
->>> c("to_char(12454, '99G999')")
-'12,454'
->>> c("to_char(1234, '9,999')")
-'1,234'
->>> c("to_char(1234, '9G999')")
-'1,234'
->>> c("to_char(45, '9,999')")
-'   45'
->>> c("to_char(45, '0,000')")
-'0,045'
->>> c("to_char(1234567, '9,999,999')")
-'1,234,567'
->>> c("to_char(1234567, '9999,999')")
-'1234,567'
->>> c("to_char(12454, '99G99,9')")
-'12,45,4'
->>> c("to_char(1234.56, '9,999.99')")
-'1,234.56'
-
-Currency sign.
-
->>> c("to_char(78.12, '$99.99')")
-'$78.12'
->>> c("to_char(78.12, '$00.00')")
-'$78.12'
->>> c("to_char(-78.12, '$99.99')")
-'$78.12'
->>> c("to_char(0.12, '$.99')")
-'$.12'
-
-'S' prints '+' or '-'; 'MI' prints '-' or a space. Signs stay adjacent to
-the digits when the digit sequence is space-padded.
-
->>> c("to_char(454, 'S999')")
-'+454'
->>> c("to_char(-454, 'S999')")
-'-454'
->>> c("to_char(454, '999S')")
-'454+'
->>> c("to_char(-454, '999S')")
-'454-'
->>> c("to_char(-12454.8, '99G999D9S')")
-'12,454.8-'
->>> c("to_char(-1, 'S9999')")
-'   -1'
->>> c("to_char(1, 'S9999')")
-'   +1'
->>> c("to_char(-1, 'S0000')")
-'-0001'
->>> c("to_char(-454, 'S$999')")
-'-$454'
->>> c("to_char(-78.12, 'S$99.99')")
-'-$78.12'
->>> c("to_char(-78.12, '$99.99S')")
-'$78.12-'
->>> c("to_char(454, 'MI999')")
-' 454'
->>> c("to_char(-454, 'MI999')")
-'-454'
->>> c("to_char(454, '999MI')")
-'454 '
->>> c("to_char(-454, '999MI')")
-'454-'
->>> c("to_char(-1, 'MI9999')")
-'   -1'
->>> c("to_char(-1, '9999MI')")
-'   1-'
-
-'PR' wraps negative values in angle brackets; positive values get two
-trailing spaces.
-
->>> c("to_char(454, '999PR')")
-'454  '
->>> c("to_char(-454, '999PR')")
-'<454>'
->>> c("to_char(-454, '9999PR')")
-' <454>'
->>> c("to_char(-4, '9999PR')")
-'   <4>'
->>> c("to_char(-78.12, '$99.99PR')")
-'<$78.12>'
->>> c("to_char(-454, 'S999PR')")
-'<-454>'
-
-Sign and decimal point interaction. A decimal point left dangling at the
-end of the digits is replaced by a space.
-
->>> c("to_char(-0.1, 'S9.9')")
-' -.1'
->>> c("to_char(-0.1, '9.9S')")
-'0.1-'
->>> c("to_char(-0.1, '9.9MI')")
-'0.1-'
->>> c("to_char(0.1, '9.9S')")
-'0.1+'
->>> c("to_char(78, '99.S')")
-'78+ '
->>> c("to_char(-78, '99.S')")
-'78- '
->>> c("to_char(-78, '99.PR')")
-'<78> '
-
-Without a sign token, the sign of a negative value is dropped.
-
->>> c("to_char(-454, '999')")
-'454'
->>> c("to_char(-12.34, '99.99')")
-'12.34'
-
-Trailing zeros of the decimal value beyond its significant digits are
-dropped before formatting, and the format pads with zeros after the
-decimal point.
-
->>> c("to_char(CAST(78.12 AS DECIMAL(10,4)), '99.99')")
-'78.12'
->>> c("to_char(CAST(78.1 AS DECIMAL(10,4)), '99.99')")
-'78.10'
->>> c("to_char(CAST(78 AS DECIMAL(10,4)), '99.99')")
-'78.00'
->>> c("to_char(CAST(0.5 AS DECIMAL(10,4)), '99.99')")
-' 0.50'
->>> c("to_char(454.00, '000D00')")
-'454.00'
->>> c("to_char(CAST(12454.8 AS DECIMAL(20,10)), '99G999D9S')")
-'12,454.8+'
-
-Floating-point inputs are converted like Spark casts them to decimals:
-DOUBLE to DECIMAL(30, 15) and FLOAT to DECIMAL(14, 7), based on the
-shortest decimal representation of the (widened) double value. Note that
-`CAST(78.12 AS FLOAT)` widens to the double 78.12000274658203, whose seven
-fractional digits 1200027 overflow the two fractional digits in the format.
-
->>> c("to_char(78.12D, '$99.99')")
-'$78.12'
->>> c("to_char(CAST(0.001 AS DOUBLE), '9.999')")
-'0.001'
->>> c("to_char(CAST(0.1 AS DOUBLE), '9.9')")
-'0.1'
->>> c("to_char(CAST(-12454.8 AS DOUBLE), '99G999D9S')")
-'12,454.8-'
->>> c("to_char(CAST(0.12345678901234567 AS DOUBLE), '9.99999999999999999')")
-'0.12345678901234600'
->>> c("to_char(CAST(1.5 AS FLOAT), '9.9')")
-'1.5'
->>> c("to_char(CAST(78.12 AS FLOAT), '99.99')")
-'##.##'
-
-Integer inputs.
-
->>> c("to_char(CAST(127 AS TINYINT), '999')")
-'127'
->>> c("to_char(CAST(-32768 AS SMALLINT), 'S99999')")
-'-32768'
->>> c("to_char(CAST(2147483647 AS INT), '9999999999')")
-'2147483647'
->>> c("to_char(CAST(9223372036854775807 AS BIGINT), '9999999999999999999')")
-'9223372036854775807'
-
-String inputs are parsed as decimals.
-
->>> c("to_char('78.12', '99.99')")
-'78.12'
->>> c("to_char('454', '999')")
-'454'
->>> c("to_char('-454', 'S999')")
-'-454'
-
-NULL handling: a NULL value or a NULL format yields NULL.
-
->>> c("to_char(CAST(NULL AS DECIMAL(5,2)), '999')") is None
-True
->>> c("to_char(CAST(NULL AS INT), '999')") is None
-True
->>> c("to_char(454, CAST(NULL AS STRING))") is None
-True
-
-Large decimals beyond 64-bit unscaled values.
-
->>> c("to_char(CAST('12345678901234567890123456789012345678' AS DECIMAL(38,0)), '99999999999999999999999999999999999999')")
-'12345678901234567890123456789012345678'
->>> c("to_char(CAST('1234567890.0987654321' AS DECIMAL(38,18)), '9999999999.999999999999999999')")
-'1234567890.098765432100000000'
-
-The format string is case-insensitive.
-
->>> c("to_char(-12454.8, '99g999d9s')")
-'12,454.8-'
->>> c("to_char(-454, '999pr')")
-'<454>'
->>> c("to_char(-454, 'mi999')")
-'-454'
->>> c("to_char(78.12, '$99d99')")
-'$78.12'
-
-Invalid number formats raise an error.
-
->>> err("to_char(454, '')")
-error
->>> err("to_char(454, 'SS999')")
-error
->>> err("to_char(454, '9.9.9')")
-error
->>> err("to_char(454, '$$999')")
-error
->>> err("to_char(454, '9$9')")
-error
->>> err("to_char(454, '.99$')")
-error
->>> err("to_char(78.12, '99.99$')")
-error
->>> err("to_char(454, ',999')")
-error
->>> err("to_char(454, '999,')")
-error
->>> err("to_char(454, '9,,99')")
-error
->>> err("to_char(454, '99.9,9')")
-error
->>> err("to_char(454, 'PR999')")
-error
->>> err("to_char(454, 'MI999MI')")
-error
->>> err("to_char(454, 'x99')")
-error
->>> err("to_char(454, '99x')")
-error
->>> err("to_char(12.34, '99v99')")
-error
->>> err("to_char(454, 'S')")
-error
->>> err("to_char(454, '$')")
-error
->>> err("to_char(454, 'MI')")
-error
->>> err("to_char(454, 'PR')")
-error
->>> err("to_char(454, '999PRMI')")
-error
-
-Datetime inputs use the Spark datetime pattern syntax. JVM Spark only
-supports this for 4.x, so these examples are skipped on JVM Spark; the
-expected values match JVM Spark 4.1.1.
-
->>> c("to_char(date'2016-04-08', 'y')")  # doctest: +SAIL_ONLY
-'2016'
->>> c("to_char(date'2022-04-08', 'yyyy-MM-dd')")  # doctest: +SAIL_ONLY
-'2022-04-08'
->>> c("to_char(timestamp'2016-04-08 13:14:15', 'yyyy-MM-dd HH:mm:ss')")  # doctest: +SAIL_ONLY
-'2016-04-08 13:14:15'
->>> c("to_char(date'2022-04-08', 'MMM')")  # doctest: +SAIL_ONLY
-'Apr'
->>> c("to_char(date'2022-04-08', 'E')")  # doctest: +SAIL_ONLY
-'Fri'
->>> c("to_char(timestamp_ntz'2016-04-08 13:14:15', 'yyyy MM dd')")  # doctest: +SAIL_ONLY
-'2016 04 08'
->>> c("to_char(CAST(NULL AS DATE), 'yyyy')") is None  # doctest: +SAIL_ONLY
-True
-
-Binary inputs support the 'base64', 'hex', and 'utf-8' formats
-(case-insensitive, surrounding whitespace ignored). JVM Spark only
-supports this for 4.x, so these examples are skipped on JVM Spark; the
-expected values match JVM Spark 4.1.1.
-
->>> c("to_char(encode('Spark SQL', 'utf-8'), 'base64')")  # doctest: +SAIL_ONLY
-'U3BhcmsgU1FM'
->>> c("to_char(encode('Spark SQL', 'utf-8'), 'hex')")  # doctest: +SAIL_ONLY
-'537061726B2053514C'
->>> c("to_char(encode('abc', 'utf-8'), 'utf-8')")  # doctest: +SAIL_ONLY
-'abc'
->>> c("to_char(encode('Spark SQL', 'utf-8'), 'BASE64')")  # doctest: +SAIL_ONLY
-'U3BhcmsgU1FM'
->>> c("to_char(encode('Spark SQL', 'utf-8'), ' hex ')")  # doctest: +SAIL_ONLY
-'537061726B2053514C'
->>> c("to_char(encode('abc', 'utf-8'), 'UTF-8')")  # doctest: +SAIL_ONLY
-'abc'
->>> err("to_char(encode('abc', 'utf-8'), 'foo')")  # doctest: +SAIL_ONLY
-error
->>> err("to_char(encode('abc', 'utf-8'), CAST(NULL AS STRING))")  # doctest: +SAIL_ONLY
-error
diff --git a/python/pysail/tests/spark/function/test_to_date.txt b/python/pysail/tests/spark/function/test_to_date.txt
deleted file mode 100644
index 549d72ea9..000000000
--- a/python/pysail/tests/spark/function/test_to_date.txt
+++ /dev/null
@@ -1,37 +0,0 @@
-# Timestamp args not covered by Spark tests
->>> from pyspark.sql import functions as sf
->>> spark.conf.set("spark.sql.session.timeZone", "UTC")
->>> spark.sql("SELECT to_date(TIMESTAMP_NTZ '2025-11-02 23:30:45.123456')").show(truncate=False)
-+---------------------------------------------------+
-|to_date(TIMESTAMP_NTZ '2025-11-02 23:30:45.123456')|
-+---------------------------------------------------+
-|2025-11-02                                         |
-+---------------------------------------------------+
->>> spark.sql("SELECT to_date(TIMESTAMP_LTZ '2025-11-02 23:30:45.123456')").show(truncate=False)
-+-----------------------------------------------+
-|to_date(TIMESTAMP '2025-11-02 23:30:45.123456')|
-+-----------------------------------------------+
-|2025-11-02                                     |
-+-----------------------------------------------+
->>> spark.sql("SELECT to_date(TIMESTAMP_LTZ '2025-11-02 23:30:45.123456 America/New_York')").show(truncate=False)
-+-----------------------------------------------+
-|to_date(TIMESTAMP '2025-11-03 04:30:45.123456')|
-+-----------------------------------------------+
-|2025-11-03                                     |
-+-----------------------------------------------+
->>> spark.sql("SELECT to_date(TIMESTAMP '2025-11-03 23:30:45.123456', 'invalid_format')").show(truncate=False)
-+---------------------------------------------------------------+
-|to_date(TIMESTAMP '2025-11-03 23:30:45.123456', invalid_format)|
-+---------------------------------------------------------------+
-|2025-11-03                                                     |
-+---------------------------------------------------------------+
->>> df = spark.createDataFrame([("2025-11-02 23:30:45.123456",), ("2025-11-02 23:30:45.123456-08:00",), ("2025-11-02 23:30:45.123456+01:00",)], ["ts"])
->>> df = df.select("ts", sf.expr("CAST(ts AS TIMESTAMP_NTZ)").alias("ts_ntz"), sf.expr("CAST(ts AS TIMESTAMP_LTZ)").alias("ts_ltz"))
->>> df.select("ts", "ts_ntz", "ts_ltz", sf.to_date("ts_ntz").alias("date_ntz"), sf.to_date("ts_ltz").alias("date_ltz")).show(truncate=False)
-+--------------------------------+--------------------------+--------------------------+----------+----------+
-|ts                              |ts_ntz                    |ts_ltz                    |date_ntz  |date_ltz  |
-+--------------------------------+--------------------------+--------------------------+----------+----------+
-|2025-11-02 23:30:45.123456      |2025-11-02 23:30:45.123456|2025-11-02 23:30:45.123456|2025-11-02|2025-11-02|
-|2025-11-02 23:30:45.123456-08:00|2025-11-02 23:30:45.123456|2025-11-03 07:30:45.123456|2025-11-02|2025-11-03|
-|2025-11-02 23:30:45.123456+01:00|2025-11-02 23:30:45.123456|2025-11-02 22:30:45.123456|2025-11-02|2025-11-02|
-+--------------------------------+--------------------------+--------------------------+----------+----------+
diff --git a/python/pysail/tests/spark/function/test_to_number.txt b/python/pysail/tests/spark/function/test_to_number.txt
deleted file mode 100644
index a9b226cd0..000000000
--- a/python/pysail/tests/spark/function/test_to_number.txt
+++ /dev/null
@@ -1,83 +0,0 @@
->>> spark.sql("SELECT to_number('<-$12,345.67>', 'S$999,099.99PR') AS result, typeof(to_number('<-$12,345.67>', 'S$999,099.99PR')) AS type").show(truncate=False)
-+--------+------------+
-|result  |type        |
-+--------+------------+
-|12345.67|decimal(8,2)|
-+--------+------------+
-
->>> spark.sql("SELECT to_number('<$1,212,345.67>', 'S$0,000,000.99PR') AS result, typeof(to_number('<$1,212,345.67>', 'S$0,000,000.99PR')) AS type").show(truncate=False)
-+-----------+------------+
-|result     |type        |
-+-----------+------------+
-|-1212345.67|decimal(9,2)|
-+-----------+------------+
-
->>> spark.sql("SELECT to_number('$345', 'S$999,099.99') AS result, typeof(to_number('$345', 'S$999,099.99')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|345.00|decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT to_number('$045', 'S$999,099.99') AS result, typeof(to_number('$045', 'S$999,099.99')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|45.00 |decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT to_number('<1234>', '999999PR') AS result, typeof(to_number('<1234>', '999999PR')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|-1234 |decimal(6,0)|
-+------+------------+
-
->>> spark.sql("SELECT to_number('12,454.8-', '99,999.9S') AS result, typeof(to_number('12,454.8-', '99,999.9S')) AS type").show(truncate=False)
-+--------+------------+
-|result  |type        |
-+--------+------------+
-|-12454.8|decimal(6,1)|
-+--------+------------+
-
->>> spark.sql("SELECT to_number('<-$123,456.32>', 'S$999,999.999999PR') AS result, typeof(to_number('<-$123,456.32>', 'S$999,999.999999PR')) AS type").show(truncate=False)
-+-------------+-------------+
-|result       |type         |
-+-------------+-------------+
-|123456.320000|decimal(12,6)|
-+-------------+-------------+
-
->>> spark.sql("SELECT to_number('$123,456.32', 'MI$999,999.99S') AS result, typeof(to_number('$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False)
-+---------+------------+
-|result   |type        |
-+---------+------------+
-|123456.32|decimal(8,2)|
-+---------+------------+
-
->>> spark.sql("SELECT to_number('$123,456.32-', 'MI$999,999.99S') AS result, typeof(to_number('$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False)
-+----------+------------+
-|result    |type        |
-+----------+------------+
-|-123456.32|decimal(8,2)|
-+----------+------------+
-
->>> spark.sql("SELECT to_number('-$123,456.32-', 'MI$999,999.99S') AS result, typeof(to_number('-$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False)
-+---------+------------+
-|result   |type        |
-+---------+------------+
-|123456.32|decimal(8,2)|
-+---------+------------+
-
->>> spark.sql("SELECT to_number('-$123,456.32+', 'MI$999,999.99S') AS result, typeof(to_number('-$123,456.32+', 'MI$999,999.99S')) AS type").show(truncate=False)
-+----------+------------+
-|result    |type        |
-+----------+------------+
-|-123456.32|decimal(8,2)|
-+----------+------------+
-
->>> spark.sql("SELECT to_number('-$123,456.32', 'MI$999,999.99S') AS result, typeof(to_number('-$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False)
-+----------+------------+
-|result    |type        |
-+----------+------------+
-|-123456.32|decimal(8,2)|
-+----------+------------+
diff --git a/python/pysail/tests/spark/function/test_to_timestamp_ntz.txt b/python/pysail/tests/spark/function/test_to_timestamp_ntz.txt
deleted file mode 100644
index cffefe979..000000000
--- a/python/pysail/tests/spark/function/test_to_timestamp_ntz.txt
+++ /dev/null
@@ -1,32 +0,0 @@
->>> from datetime import datetime
->>> from zoneinfo import ZoneInfo
->>> from pyspark.sql import functions as F, types as T
->>>
->>> spark.conf.set("spark.sql.session.timeZone", "Europe/Amsterdam")
->>>
->>> df = spark.createDataFrame(
-...     [
-...         (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Europe/Amsterdam")),),
-...         (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Asia/Shanghai")),)
-...     ],
-...     T.StructType([T.StructField("ts", T.TimestampType(), True)]),
-... )
->>> df.show()
-+-------------------+
-|                 ts|
-+-------------------+
-|2023-01-01 10:00:00|
-|2023-01-01 03:00:00|
-+-------------------+
-
->>> df.select(F.to_timestamp_ntz("ts")).show()
-+--------------------+
-|to_timestamp_ntz(ts)|
-+--------------------+
-| 2023-01-01 10:00:00|
-| 2023-01-01 03:00:00|
-+--------------------+
-
->>> df.select(F.to_timestamp_ntz("ts")).printSchema()
-root
- |-- to_timestamp_ntz(ts): timestamp_ntz (nullable = true)
diff --git a/python/pysail/tests/spark/function/test_to_varchar.txt b/python/pysail/tests/spark/function/test_to_varchar.txt
deleted file mode 100644
index 906217b1a..000000000
--- a/python/pysail/tests/spark/function/test_to_varchar.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-Tests for `to_varchar`.
-
-`to_varchar` follows the same formatting behavior as the underlying formatter.
-
->>> from pyspark.sql.functions import lit, to_varchar
-
->>> df = spark.createDataFrame([(78.12,)], ["e"])
->>> df.select(to_varchar(df.e, lit("$99.99")).alias("r")).collect()
-[Row(r='$78.12')]
-
->>> def c(expr):
-...     return spark.sql(f"SELECT {expr} AS r").collect()[0]["r"]
->>> def err(expr):
-...     try:
-...         spark.sql(f"SELECT {expr} AS r").collect()
-...         print("no error")
-...     except Exception:
-...         print("error")
-
->>> c("to_varchar(454, '999')")
-'454'
->>> c("to_varchar(78.12, '$99.99')")
-'$78.12'
->>> c("to_varchar(-12454.8, '99G999D9S')")
-'12,454.8-'
->>> err("to_varchar(454, 'PR999')")
-error
->>> c("to_varchar(encode('Spark SQL', 'utf-8'), 'hex')")  # doctest: +SAIL_ONLY
-'537061726B2053514C'
diff --git a/python/pysail/tests/spark/function/test_try_add.txt b/python/pysail/tests/spark/function/test_try_add.txt
deleted file mode 100644
index a5ee04d4d..000000000
--- a/python/pysail/tests/spark/function/test_try_add.txt
+++ /dev/null
@@ -1,73 +0,0 @@
->>> from pyspark.sql.types import StructType, StructField, IntegerType
->>> import pyspark.sql.functions as sf
-
->>> schema = StructType([StructField("birth", IntegerType(), True),StructField("age", IntegerType(), True),])
->>> df = spark.createDataFrame([(1982, 15),(1990, 2),(None, 10),(2147483647, 1),(-2147483648,-1),],schema=schema)
->>> df.select("*", sf.try_add("birth", "age").alias("birth_plus_age")).show()
-+-----------+---+--------------+
-|      birth|age|birth_plus_age|
-+-----------+---+--------------+
-|       1982| 15|          1997|
-|       1990|  2|          1992|
-|       NULL| 10|          NULL|
-| 2147483647|  1|          NULL|
-|-2147483648| -1|          NULL|
-+-----------+---+--------------+
-
->>> df = spark.sql("""
-... SELECT
-...   try_add(DATE '2015-09-30', 1) as d1,
-...   try_add(DATE '2000-01-01', 366) as d2,
-...   try_add(DATE '2021-01-01', 1) as d3,
-...   try_add(NULL, 100) as d4
-... """)
->>> df.show()
-+----------+----------+----------+----+
-|        d1|        d2|        d3|  d4|
-+----------+----------+----------+----+
-|2015-10-01|2001-01-01|2021-01-02|NULL|
-+----------+----------+----------+----+
-
->>> df = spark.sql("""
-... SELECT
-...   try_add(DATE '2015-01-31', INTERVAL 1 MONTH) as d1,
-...   try_add(DATE '2020-02-29', INTERVAL 12 MONTH) as d2,
-...   try_add(NULL, INTERVAL 3 MONTH) as d3
-... """)
->>> df.show()
-+----------+----------+----+
-|        d1|        d2|  d3|
-+----------+----------+----+
-|2015-02-28|2021-02-28|NULL|
-+----------+----------+----+
-
->>> df = spark.sql("""
-... SELECT
-...   try_add(DATE '2000-07-31', INTERVAL -1 MONTH) as d1,
-...   try_add(DATE '2021-01-31', INTERVAL -1 MONTH) as d2
-... """)
->>> df.show()
-+----------+----------+
-|        d1|        d2|
-+----------+----------+
-|2000-06-30|2020-12-31|
-+----------+----------+
-
->>> df = spark.sql("""
-... SELECT
-...   try_add(INTERVAL '1' YEAR, INTERVAL '2' YEAR) as result
-... """)
->>> # INTERVAL '3' YEAR            <-- sail
->>> # INTERVAL '3-0' YEAR TO MONTH <-- PySpark
-
->>> spark.conf.set("spark.sql.session.timeZone", "UTC")
->>> df = spark.sql("""
-... SELECT
-...   try_add(TIMESTAMP '2021-01-01 00:00:00', INTERVAL 1 DAY) as result
-... """)
->>> df.show(truncate=False)
-+-------------------+
-|result             |
-+-------------------+
-|2021-01-02 00:00:00|
-+-------------------+
diff --git a/python/pysail/tests/spark/function/test_try_divide.txt b/python/pysail/tests/spark/function/test_try_divide.txt
deleted file mode 100644
index dfd644a30..000000000
--- a/python/pysail/tests/spark/function/test_try_divide.txt
+++ /dev/null
@@ -1,119 +0,0 @@
->>> from pyspark.sql import functions as sf
-
->>> df = spark.createDataFrame([(6000, 15), (1990, 2)], ["a", "b"])
->>> df.select(sf.try_divide(df.a, df.b).alias('r')).show()
-+-----+
-|    r|
-+-----+
-|400.0|
-|995.0|
-+-----+
-
->>> spark.range(1).select(sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(2))).show()
-+-------------------------------------------------+
-|try_divide(make_interval(0, 0, 0, 1, 0, 0, 0), 2)|
-+-------------------------------------------------+
-|                                         12 hours|
-+-------------------------------------------------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(hours=sf.lit(10)), sf.lit(4)).alias("result")
-... ).show(truncate=False)
-+------------------+
-|result            |
-+------------------+
-|2 hours 30 minutes|
-+------------------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(days=sf.lit(3), hours=sf.lit(12)), sf.lit(3)).alias("result")
-... ).show(truncate=False)
-+--------------+
-|result        |
-+--------------+
-|1 days 4 hours|
-+--------------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(secs=sf.lit(10.5)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+------------+
-|result      |
-+------------+
-|5.25 seconds|
-+------------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(weeks=sf.lit(1)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+---------------+
-|result         |
-+---------------+
-|3 days 12 hours|
-+---------------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(None).cast("int")).alias("result")
-... ).show(truncate=False)
-+------+
-|result|
-+------+
-|NULL  |
-+------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(0)).alias("result")
-... ).show(truncate=False)
-+------+
-|result|
-+------+
-|NULL  |
-+------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(days=sf.lit(1)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+--------+
-|result  |
-+--------+
-|12 hours|
-+--------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(days=sf.lit(-1)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+---------+
-|result   |
-+---------+
-|-12 hours|
-+---------+
-
->>> spark.range(1).select(
-...     sf.try_divide(sf.make_interval(days=sf.lit(1), mins=sf.lit(90)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+-------------------+
-|result             |
-+-------------------+
-|12 hours 45 minutes|
-+-------------------+
-
->>> spark.sql("SELECT try_divide(make_ym_interval(1, 6), 2) AS result").show(truncate=False)
-+----------------------------+
-|result                      |
-+----------------------------+
-|INTERVAL '0-9' YEAR TO MONTH|
-+----------------------------+
-
->>> spark.range(1).selectExpr("try_divide(make_ym_interval(1, 6), 2) AS result").show(truncate=False)
-+----------------------------+
-|result                      |
-+----------------------------+
-|INTERVAL '0-9' YEAR TO MONTH|
-+----------------------------+
-
->>> spark.range(1).select(sf.try_divide(sf.make_interval(secs=sf.lit(1)), sf.lit(2))).show()
-+-------------------------------------------------+
-|try_divide(make_interval(0, 0, 0, 0, 0, 0, 1), 2)|
-+-------------------------------------------------+
-|                                      0.5 seconds|
-+-------------------------------------------------+
diff --git a/python/pysail/tests/spark/function/test_try_mod.txt b/python/pysail/tests/spark/function/test_try_mod.txt
deleted file mode 100644
index fdb952681..000000000
--- a/python/pysail/tests/spark/function/test_try_mod.txt
+++ /dev/null
@@ -1,188 +0,0 @@
->>> spark.sql("SELECT try_mod(CAST(3.0  AS DECIMAL(2,1)), CAST(2.00 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|1.00|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(12.34 AS DECIMAL(4,2)), CAST(5.67 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|1.00|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(1.2 AS DECIMAL(2,1)), CAST(0.03 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|0.00|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.00 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|NULL|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(0.00 AS DECIMAL(3,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|0.00|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(9.99 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|1.23|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(5.00 AS DECIMAL(3,2)), CAST(5.00 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|0.00|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(9.9 AS DECIMAL(38,1)), CAST(0.000000000000000001 AS DECIMAL(18,18))) AS r").show()
-+--------------------+
-|                   r|
-+--------------------+
-|0.000000000000000000|
-+--------------------+
-
->>> spark.sql("SELECT try_mod(CAST(99999999999999999999999999999999999999 AS DECIMAL(38,0)), CAST(10 AS DECIMAL(2,0))) AS r").show()
-+---+
-|  r|
-+---+
-|  9|
-+---+
-
->>> spark.sql("SELECT try_mod(3, CAST(2.00 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|1.00|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), 2) AS r").show()
-+----+
-|   r|
-+----+
-|1.00|
-+----+
-
->>> spark.sql("SELECT try_mod(NULL, CAST(2.00 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|NULL|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), NULL) AS r").show()
-+----+
-|   r|
-+----+
-|NULL|
-+----+
-
->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.000000000001 AS DECIMAL(13,12))) AS r").show()
-+--------------+
-|             r|
-+--------------+
-|0.000000000000|
-+--------------+
-
->>> spark.sql("SELECT try_mod(CAST(10.00 AS DECIMAL(4,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").show()
-+----+
-|   r|
-+----+
-|0.00|
-+----+
-
->>> spark.sql("""SELECT
-...   try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS p_p,
-...   try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS n_p,
-...   try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS p_n,
-...   try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS n_n
-... """).show()
-+---+----+---+----+
-|p_p| n_p|p_n| n_n|
-+---+----+---+----+
-|1.5|-1.5|1.5|-1.5|
-+---+----+---+----+
-
->>> spark.sql("SELECT try_mod(CAST(3.0  AS DECIMAL(2,1)), CAST(2.00 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(12.34 AS DECIMAL(4,2)), CAST(5.67 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(1.2 AS DECIMAL(2,1)), CAST(0.03 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.00 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(0.00 AS DECIMAL(3,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(9.99 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(5.00 AS DECIMAL(3,2)), CAST(5.00 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(9.9 AS DECIMAL(38,1)), CAST(0.000000000000000001 AS DECIMAL(18,18))) AS r").printSchema()
-root
- |-- r: decimal(18,18) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(99999999999999999999999999999999999999 AS DECIMAL(38,0)), CAST(10 AS DECIMAL(2,0))) AS r").printSchema()
-root
- |-- r: decimal(2,0) (nullable = true)
-
->>> spark.sql("SELECT try_mod(3, CAST(2.00 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), 2) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(NULL, CAST(2.00 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(3.00 AS DECIMAL(3,2)), NULL) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(1.23 AS DECIMAL(3,2)), CAST(0.000000000001 AS DECIMAL(13,12))) AS r").printSchema()
-root
- |-- r: decimal(13,12) (nullable = true)
-
->>> spark.sql("SELECT try_mod(CAST(10.00 AS DECIMAL(4,2)), CAST(2.50 AS DECIMAL(3,2))) AS r").printSchema()
-root
- |-- r: decimal(3,2) (nullable = true)
-
->>> spark.sql("""SELECT
-... try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS p_p,
-... try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST( 2.0 AS DECIMAL(2,1))) AS n_p,
-... try_mod(CAST( 7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS p_n,
-... try_mod(CAST(-7.5 AS DECIMAL(3,1)), CAST(-2.0 AS DECIMAL(2,1))) AS n_n
-... """).printSchema()
-root
- |-- p_p: decimal(2,1) (nullable = true)
- |-- n_p: decimal(2,1) (nullable = true)
- |-- p_n: decimal(2,1) (nullable = true)
- |-- n_n: decimal(2,1) (nullable = true)
diff --git a/python/pysail/tests/spark/function/test_try_multiply.txt b/python/pysail/tests/spark/function/test_try_multiply.txt
deleted file mode 100644
index 6437ad75d..000000000
--- a/python/pysail/tests/spark/function/test_try_multiply.txt
+++ /dev/null
@@ -1,96 +0,0 @@
->>> from pyspark.sql import functions as sf
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(days=sf.lit(1)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+------+
-|result|
-+------+
-|2 days|
-+------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(hours=sf.lit(10)), sf.lit(4)).alias("result")
-... ).show(truncate=False)
-+--------+
-|result  |
-+--------+
-|40 hours|
-+--------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(days=sf.lit(3), hours=sf.lit(12)), sf.lit(3)).alias("result")
-... ).show(truncate=False)
-+---------------+
-|result         |
-+---------------+
-|9 days 36 hours|
-+---------------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(secs=sf.lit(10.5)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+----------+
-|result    |
-+----------+
-|21 seconds|
-+----------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(weeks=sf.lit(1)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+-------+
-|result |
-+-------+
-|14 days|
-+-------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(days=sf.lit(1)), sf.lit(None).cast("int")).alias("result")
-... ).show(truncate=False)
-+------+
-|result|
-+------+
-|NULL  |
-+------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(days=sf.lit(1)), sf.lit(0)).alias("result")
-... ).show(truncate=False)
-+---------+
-|result   |
-+---------+
-|0 seconds|
-+---------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(days=sf.lit(-1)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+-------+
-|result |
-+-------+
-|-2 days|
-+-------+
-
->>> spark.range(1).select(
-...     sf.try_multiply(sf.make_interval(days=sf.lit(1), mins=sf.lit(90)), sf.lit(2)).alias("result")
-... ).show(truncate=False)
-+--------------+
-|result        |
-+--------------+
-|2 days 3 hours|
-+--------------+
-
->>> spark.sql("SELECT try_multiply(make_ym_interval(1, 6), 2) AS result").show(truncate=False)
-+----------------------------+
-|result                      |
-+----------------------------+
-|INTERVAL '3-0' YEAR TO MONTH|
-+----------------------------+
-
->>> spark.range(1).selectExpr("try_multiply(make_ym_interval(1, 6), 2) AS result").show(truncate=False)
-+----------------------------+
-|result                      |
-+----------------------------+
-|INTERVAL '3-0' YEAR TO MONTH|
-+----------------------------+
diff --git a/python/pysail/tests/spark/function/test_try_parse_url.txt b/python/pysail/tests/spark/function/test_try_parse_url.txt
deleted file mode 100644
index 7b0d720e6..000000000
--- a/python/pysail/tests/spark/function/test_try_parse_url.txt
+++ /dev/null
@@ -1,69 +0,0 @@
->>> spark.sql("SELECT try_parse_url('https://example.com/a?x=1', 'QUERY', 'x') AS result, typeof(try_parse_url('https://example.com/a?x=1', 'QUERY', 'x')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|1     |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('www.example.com/path?x=1', 'HOST') AS result, typeof(try_parse_url('www.example.com/path?x=1', 'HOST')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|NULL  |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('https://example.com/?a=1', 'QUERY', 'b') AS result, typeof(try_parse_url('https://example.com/?a=1', 'QUERY', 'b')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|NULL  |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('https://example.com/path#frag', 'REF') AS result, typeof(try_parse_url('https://example.com/path#frag', 'REF')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|frag  |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO') AS result, typeof(try_parse_url('ftp://user:pwd@ftp.example.com:21/files', 'USERINFO')) AS type").show(truncate=False)
-+--------+------+
-|result  |type  |
-+--------+------+
-|user:pwd|string|
-+--------+------+
-
->>> spark.sql("SELECT try_parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST') AS result, typeof(try_parse_url('http://[2001:db8::2]:8080/index.html?ok=1', 'HOST')) AS type").show(truncate=False)
-+-------------+------+
-|result       |type  |
-+-------------+------+
-|[2001:db8::2]|string|
-+-------------+------+
-
->>> spark.sql("SELECT try_parse_url('notaurl', 'HOST') AS result, typeof(try_parse_url('notaurl', 'HOST')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|NULL  |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('https://example.com', 'PATH') AS result, typeof(try_parse_url('https://example.com', 'PATH')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|      |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL') AS result, typeof(try_parse_url('https://example.com/a/b?x=1&y=2#frag', 'PROTOCOL')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|https |string|
-+------+------+
-
->>> spark.sql("SELECT try_parse_url('https://ex.com/?Tag=ok', 'QUERY', 'tag') AS result, typeof(try_parse_url('https://ex.com/?Tag=ok', 'QUERY', 'tag')) AS type").show(truncate=False)
-+------+------+
-|result|type  |
-+------+------+
-|NULL  |string|
-+------+------+
\ No newline at end of file
diff --git a/python/pysail/tests/spark/function/test_try_to_number.txt b/python/pysail/tests/spark/function/test_try_to_number.txt
deleted file mode 100644
index ab978ce31..000000000
--- a/python/pysail/tests/spark/function/test_try_to_number.txt
+++ /dev/null
@@ -1,153 +0,0 @@
->>> spark.sql("SELECT try_to_number('<-$12,345.67>', 'S$999,099.99PR') AS result, typeof(try_to_number('<-$12,345.67>', 'S$999,099.99PR')) AS type").show(truncate=False)
-+--------+------------+
-|result  |type        |
-+--------+------------+
-|12345.67|decimal(8,2)|
-+--------+------------+
-
->>> spark.sql("SELECT try_to_number('<$1,212,345.67>', 'S$0,000,000.99PR') AS result, typeof(try_to_number('<$1,212,345.67>', 'S$0,000,000.99PR')) AS type").show(truncate=False)
-+-----------+------------+
-|result     |type        |
-+-----------+------------+
-|-1212345.67|decimal(9,2)|
-+-----------+------------+
-
->>> spark.sql("SELECT try_to_number('$345', 'S$999,099.99') AS result, typeof(try_to_number('$345', 'S$999,099.99')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|345.00|decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('$045', 'S$999,099.99') AS result, typeof(try_to_number('$045', 'S$999,099.99')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|45.00 |decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('<1234>', '999999PR') AS result, typeof(try_to_number('<1234>', '999999PR')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|-1234 |decimal(6,0)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('12,454.8-', '99,999.9S') AS result, typeof(try_to_number('12,454.8-', '99,999.9S')) AS type").show(truncate=False)
-+--------+------------+
-|result  |type        |
-+--------+------------+
-|-12454.8|decimal(6,1)|
-+--------+------------+
-
->>> spark.sql("SELECT try_to_number('<-$123,456.32>', 'S$999,999.999999PR') AS result, typeof(try_to_number('<-$123,456.32>', 'S$999,999.999999PR')) AS type").show(truncate=False)
-+-------------+-------------+
-|result       |type         |
-+-------------+-------------+
-|123456.320000|decimal(12,6)|
-+-------------+-------------+
-
->>> spark.sql("SELECT try_to_number('$123,456.32', 'MI$999,999.99S') AS result, typeof(try_to_number('$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False)
-+---------+------------+
-|result   |type        |
-+---------+------------+
-|123456.32|decimal(8,2)|
-+---------+------------+
-
->>> spark.sql("SELECT try_to_number('$123,456.32-', 'MI$999,999.99S') AS result, typeof(try_to_number('$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False)
-+----------+------------+
-|result    |type        |
-+----------+------------+
-|-123456.32|decimal(8,2)|
-+----------+------------+
-
->>> spark.sql("SELECT try_to_number('-$123,456.32-', 'MI$999,999.99S') AS result, typeof(try_to_number('-$123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False)
-+---------+------------+
-|result   |type        |
-+---------+------------+
-|123456.32|decimal(8,2)|
-+---------+------------+
-
->>> spark.sql("SELECT try_to_number('-$123,456.32+', 'MI$999,999.99S') AS result, typeof(try_to_number('-$123,456.32+', 'MI$999,999.99S')) AS type").show(truncate=False)
-+----------+------------+
-|result    |type        |
-+----------+------------+
-|-123456.32|decimal(8,2)|
-+----------+------------+
-
->>> spark.sql("SELECT try_to_number('-$123,456.32', 'MI$999,999.99S') AS result, typeof(try_to_number('-$123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False)
-+----------+------------+
-|result    |type        |
-+----------+------------+
-|-123456.32|decimal(8,2)|
-+----------+------------+
-
->>> spark.sql("SELECT try_to_number('045', 'S$999,099.99') AS result, typeof(try_to_number('045', 'S$999,099.99')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('1234', '999999PR') AS result, typeof(try_to_number('1234', '999999PR')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|1234  |decimal(6,0)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('<1234>', '999999PR') AS result, typeof(try_to_number('>1234>', '999999PR')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|-1234 |decimal(6,0)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('1234>', '999999PR') AS result, typeof(try_to_number('1234>', '999999PR')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(6,0)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('12454.8-', '99,999.9S') AS result, typeof(try_to_number('12454.8-', '99,999.9S')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(6,1)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('<$1212345.67>', 'S$0,000,000.99PR') AS result, typeof(try_to_number('<$1,212,345.67>', 'S$0,000,000.99PR')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(9,2)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('-$123,456.32', 'S$999,999.999999PR') AS result, typeof(try_to_number('-$123,456.32', 'S$999,999.999999PR')) AS type").show(truncate=False)
-+--------------+-------------+
-|result        |type         |
-+--------------+-------------+
-|-123456.320000|decimal(12,6)|
-+--------------+-------------+
-
->>> spark.sql("SELECT try_to_number('-123,456.32-', 'MI$999,999.99S') AS result, typeof(try_to_number('-123,456.32-', 'MI$999,999.99S')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('+<$123,456.32+', 'MI$999,999.99S') AS result, typeof(try_to_number('+<$123,456.32+', 'MI$999,999.99S')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(8,2)|
-+------+------------+
-
->>> spark.sql("SELECT try_to_number('-123,456.32', 'MI$999,999.99S') AS result, typeof(try_to_number('-123,456.32', 'MI$999,999.99S')) AS type").show(truncate=False)
-+------+------------+
-|result|type        |
-+------+------------+
-|NULL  |decimal(8,2)|
-+------+------------+
diff --git a/python/pysail/tests/spark/function/test_try_to_timestamp.txt b/python/pysail/tests/spark/function/test_try_to_timestamp.txt
deleted file mode 100644
index 93e3001eb..000000000
--- a/python/pysail/tests/spark/function/test_try_to_timestamp.txt
+++ /dev/null
@@ -1,48 +0,0 @@
->>> from datetime import datetime
->>> from zoneinfo import ZoneInfo
->>> from pyspark.sql import functions as F, types as T
->>>
->>> spark.conf.set("spark.sql.session.timeZone", "Europe/Amsterdam")
->>>
->>> df = spark.createDataFrame(
-...     [
-...         (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Europe/Amsterdam")),),
-...         (datetime(2023, 1, 1, 10, tzinfo=ZoneInfo("Asia/Shanghai")),)
-...     ],
-...     T.StructType([T.StructField("ts", T.TimestampType(), True)]),
-... )
->>> df.show()
-+-------------------+
-|                 ts|
-+-------------------+
-|2023-01-01 10:00:00|
-|2023-01-01 03:00:00|
-+-------------------+
-
->>> spark.conf.set("spark.sql.timestampType", "TIMESTAMP_LTZ")
->>>
->>> df.select(F.try_to_timestamp("ts")).show()
-+--------------------+
-|try_to_timestamp(ts)|
-+--------------------+
-| 2023-01-01 10:00:00|
-| 2023-01-01 03:00:00|
-+--------------------+
-
->>> df.select(F.try_to_timestamp("ts")).printSchema()
-root
- |-- try_to_timestamp(ts): timestamp (nullable = true)
-
->>> spark.conf.set("spark.sql.timestampType", "TIMESTAMP_NTZ")
->>>
->>> df.select(F.try_to_timestamp("ts")).show()
-+--------------------+
-|try_to_timestamp(ts)|
-+--------------------+
-| 2023-01-01 10:00:00|
-| 2023-01-01 03:00:00|
-+--------------------+
-
->>> df.select(F.try_to_timestamp("ts")).printSchema()
-root
- |-- try_to_timestamp(ts): timestamp_ntz (nullable = true)
diff --git a/python/pysail/tests/spark/function/test_url_decode.txt b/python/pysail/tests/spark/function/test_url_decode.txt
deleted file mode 100644
index 5fb84535f..000000000
--- a/python/pysail/tests/spark/function/test_url_decode.txt
+++ /dev/null
@@ -1,20 +0,0 @@
->>> spark.sql("select url_decode('https%3A%2F%2Fspark.apache.org')").show(truncate=False)
-+------------------------------------------+
-|url_decode(https%3A%2F%2Fspark.apache.org)|
-+------------------------------------------+
-|https://spark.apache.org                  |
-+------------------------------------------+
-
->>> spark.sql("select url_decode('inva lid://user:pass@host/file\\;param?query\\;p2')").show(truncate=False)
-+---------------------------------------------------------+
-|url_decode(inva lid://user:pass@host/file;param?query;p2)|
-+---------------------------------------------------------+
-|inva lid://user:pass@host/file;param?query;p2            |
-+---------------------------------------------------------+
-
->>> spark.sql("select url_decode(null)").show(truncate=False)
-+----------------+
-|url_decode(NULL)|
-+----------------+
-|NULL            |
-+----------------+
diff --git a/python/pysail/tests/spark/function/test_url_encode.txt b/python/pysail/tests/spark/function/test_url_encode.txt
deleted file mode 100644
index c14ce7867..000000000
--- a/python/pysail/tests/spark/function/test_url_encode.txt
+++ /dev/null
@@ -1,21 +0,0 @@
->>> spark.sql("select url_encode('https://spark.apache.org')").show(truncate=False)
-+------------------------------------+
-|url_encode(https://spark.apache.org)|
-+------------------------------------+
-|https%3A%2F%2Fspark.apache.org      |
-+------------------------------------+
-
->>> spark.sql("select url_encode('inva lid://user:pass@host/file\\;param?query\\;p2')").show(truncate=False)
-+---------------------------------------------------------------+
-|url_encode(inva lid://user:pass@host/file;param?query;p2)      |
-+---------------------------------------------------------------+
-|inva+lid%3A%2F%2Fuser%3Apass%40host%2Ffile%3Bparam%3Fquery%3Bp2|
-+---------------------------------------------------------------+
-
->>> spark.sql("select url_encode(null)").show(truncate=False)
-+----------------+
-|url_encode(NULL)|
-+----------------+
-|NULL            |
-+----------------+
-
diff --git a/python/pysail/tests/spark/function/test_width_bucket.txt b/python/pysail/tests/spark/function/test_width_bucket.txt
deleted file mode 100644
index 6e080605d..000000000
--- a/python/pysail/tests/spark/function/test_width_bucket.txt
+++ /dev/null
@@ -1,48 +0,0 @@
->>> from decimal import Decimal
->>> from pyspark.sql.types import StructType, StructField, DecimalType, IntegerType
->>> from pyspark.sql.functions import width_bucket
->>> schema = StructType([
-...     StructField("v",   DecimalType(10, 2)),
-...     StructField("min", DecimalType(10, 2)),
-...     StructField("max", DecimalType(10, 2)),
-...     StructField("n",   IntegerType()),
-... ])
->>> rows = [
-...     (Decimal("0.00"),  Decimal("0.00"),  Decimal("10.00"), 5),  # v==min → 1
-...     (Decimal("2.00"),  Decimal("0.00"),  Decimal("10.00"), 5),  # width=2 → 2
-...     (Decimal("10.00"), Decimal("0.00"),  Decimal("10.00"), 5),  # v==max → 5
-...     (Decimal("10.01"), Decimal("0.00"),  Decimal("10.00"), 5),  # v>max → 6
-...     (Decimal("-0.01"), Decimal("0.00"),  Decimal("10.00"), 5),  # v>> df = spark.createDataFrame(rows, schema)
->>> df.select(width_bucket("v", "min", "max", "n")).show()
-+----------------------------+
-|width_bucket(v, min, max, n)|
-+----------------------------+
-|                           1|
-|                           2|
-|                           6|
-|                           6|
-|                           0|
-+----------------------------+
-
->>> spark.sql("SELECT width_bucket(0.0, 10.0, 0.0, 5)").show()
-+-------------------------------+
-|width_bucket(0.0, 10.0, 0.0, 5)|
-+-------------------------------+
-|                              6|
-+-------------------------------+
-
->>> spark.sql("SELECT width_bucket(10.0, 0.0, 10.0, 5)").show()
-+--------------------------------+
-|width_bucket(10.0, 0.0, 10.0, 5)|
-+--------------------------------+
-|                               6|
-+--------------------------------+
-
->>> spark.sql("SELECT width_bucket(10.0, 0.0, 0.0, 5)").show()
-+-------------------------------+
-|width_bucket(10.0, 0.0, 0.0, 5)|
-+-------------------------------+
-|                           NULL|
-+-------------------------------+
diff --git a/python/pysail/tests/spark/function/test_zeroifnull.txt b/python/pysail/tests/spark/function/test_zeroifnull.txt
deleted file mode 100644
index bb6824287..000000000
--- a/python/pysail/tests/spark/function/test_zeroifnull.txt
+++ /dev/null
@@ -1,97 +0,0 @@
->>> spark.sql("SELECT zeroifnull(NULL), typeof(zeroifnull(NULL))").show()
-+----------------+------------------------+
-|zeroifnull(NULL)|typeof(zeroifnull(NULL))|
-+----------------+------------------------+
-|               0|                     int|
-+----------------+------------------------+
-
->>> spark.sql("SELECT zeroifnull(2), typeof(zeroifnull(2))").show()
-+-------------+---------------------+
-|zeroifnull(2)|typeof(zeroifnull(2))|
-+-------------+---------------------+
-|            2|                  int|
-+-------------+---------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(NULL AS TINYINT)), typeof(zeroifnull(CAST(NULL AS TINYINT)))").show()
-+---------------------------------+-----------------------------------------+
-|zeroifnull(CAST(NULL AS TINYINT))|typeof(zeroifnull(CAST(NULL AS TINYINT)))|
-+---------------------------------+-----------------------------------------+
-|                                0|                                  tinyint|
-+---------------------------------+-----------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(5 AS TINYINT)), typeof(zeroifnull(CAST(5 AS TINYINT)))").show()
-+------------------------------+--------------------------------------+
-|zeroifnull(CAST(5 AS TINYINT))|typeof(zeroifnull(CAST(5 AS TINYINT)))|
-+------------------------------+--------------------------------------+
-|                             5|                               tinyint|
-+------------------------------+--------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(NULL AS SMALLINT)), typeof(zeroifnull(CAST(NULL AS SMALLINT)))").show()
-+----------------------------------+------------------------------------------+
-|zeroifnull(CAST(NULL AS SMALLINT))|typeof(zeroifnull(CAST(NULL AS SMALLINT)))|
-+----------------------------------+------------------------------------------+
-|                                 0|                                  smallint|
-+----------------------------------+------------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(NULL AS BIGINT)), typeof(zeroifnull(CAST(NULL AS BIGINT)))").show()
-+--------------------------------+----------------------------------------+
-|zeroifnull(CAST(NULL AS BIGINT))|typeof(zeroifnull(CAST(NULL AS BIGINT)))|
-+--------------------------------+----------------------------------------+
-|                               0|                                  bigint|
-+--------------------------------+----------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(NULL AS FLOAT)), typeof(zeroifnull(CAST(NULL AS FLOAT)))").show()
-+-------------------------------+---------------------------------------+
-|zeroifnull(CAST(NULL AS FLOAT))|typeof(zeroifnull(CAST(NULL AS FLOAT)))|
-+-------------------------------+---------------------------------------+
-|                            0.0|                                  float|
-+-------------------------------+---------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(3.14 AS FLOAT)), typeof(zeroifnull(CAST(3.14 AS FLOAT)))").show()
-+-------------------------------+---------------------------------------+
-|zeroifnull(CAST(3.14 AS FLOAT))|typeof(zeroifnull(CAST(3.14 AS FLOAT)))|
-+-------------------------------+---------------------------------------+
-|                           3.14|                                  float|
-+-------------------------------+---------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(NULL AS DOUBLE)), typeof(zeroifnull(CAST(NULL AS DOUBLE)))").show()
-+--------------------------------+----------------------------------------+
-|zeroifnull(CAST(NULL AS DOUBLE))|typeof(zeroifnull(CAST(NULL AS DOUBLE)))|
-+--------------------------------+----------------------------------------+
-|                             0.0|                                  double|
-+--------------------------------+----------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(2.718 AS DOUBLE)), typeof(zeroifnull(CAST(2.718 AS DOUBLE)))").show()
-+---------------------------------+-----------------------------------------+
-|zeroifnull(CAST(2.718 AS DOUBLE))|typeof(zeroifnull(CAST(2.718 AS DOUBLE)))|
-+---------------------------------+-----------------------------------------+
-|                            2.718|                                   double|
-+---------------------------------+-----------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(NULL AS DECIMAL(10,2))), typeof(zeroifnull(CAST(NULL AS DECIMAL(10,2))))").show()
-+---------------------------------------+-----------------------------------------------+
-|zeroifnull(CAST(NULL AS DECIMAL(10,2)))|typeof(zeroifnull(CAST(NULL AS DECIMAL(10,2))))|
-+---------------------------------------+-----------------------------------------------+
-|                                    0.0|                                         double|
-+---------------------------------------+-----------------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(CAST(42.99 AS DECIMAL(10,2))), typeof(zeroifnull(CAST(42.99 AS DECIMAL(10,2))))").show()
-+----------------------------------------+------------------------------------------------+
-|zeroifnull(CAST(42.99 AS DECIMAL(10,2)))|typeof(zeroifnull(CAST(42.99 AS DECIMAL(10,2))))|
-+----------------------------------------+------------------------------------------------+
-|                                   42.99|                                          double|
-+----------------------------------------+------------------------------------------------+
-
->>> spark.sql("SELECT zeroifnull(0), typeof(zeroifnull(0))").show()
-+-------------+---------------------+
-|zeroifnull(0)|typeof(zeroifnull(0))|
-+-------------+---------------------+
-|            0|                  int|
-+-------------+---------------------+
-
->>> spark.sql("SELECT zeroifnull(-5), typeof(zeroifnull(-5))").show()
-+-----------------+-------------------------+
-|zeroifnull((- 5))|typeof(zeroifnull((- 5)))|
-+-----------------+-------------------------+
-|               -5|                      int|
-+-----------------+-------------------------+
diff --git a/python/pysail/tests/spark/iceberg/__snapshots__/features/merge.yaml b/python/pysail/tests/spark/iceberg/__snapshots__/features/merge.yaml
new file mode 100644
index 000000000..ac2433080
--- /dev/null
+++ b/python/pysail/tests/spark/iceberg/__snapshots__/features/merge.yaml
@@ -0,0 +1,224 @@
+# serializer version: 1
+---
+name: "test_explain_keeps_merge_metadata_scans_constant_across_target_files"
+data:
+  |-
+    == Physical Plan ==
+    IcebergCommitExec(table_path=file:///iceberg_merge_many_files_plan/)
+      CoalescePartitionsExec
+        IcebergWriterExec(table_path=file:///iceberg_merge_many_files_plan/, merge_row_intents=true)
+          UnionExec
+            ProjectionExec: expr=[id@1 as id, CASE WHEN __common_expr_1@0 THEN __sail_src_value@8 ELSE value@2 END as value, __sail_file_path@3 as __sail_file_path, __sail_file_row_index@4 as __sail_file_row_index, __sail_iceberg_partition_spec_id@5 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@6 as __sail_iceberg_partition, CASE WHEN __sail_merge_target_row_present@7 IS NULL THEN 3 WHEN __common_expr_1@0 THEN 6 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+              ProjectionExec: expr=[(__sail_merge_target_row_present@6 IS NOT NULL AND __sail_merge_source_row_present@8 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_file_row_index@3 as __sail_file_row_index, __sail_iceberg_partition_spec_id@4 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@5 as __sail_iceberg_partition, __sail_merge_target_row_present@6 as __sail_merge_target_row_present, __sail_src_value@7 as __sail_src_value]
+                FilterExec: __sail_merge_target_row_present@6 IS NOT NULL
+                  ProjectionExec: expr=[id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_file_row_index@3 as __sail_file_row_index, __sail_iceberg_partition_spec_id@4 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@5 as __sail_iceberg_partition, __sail_merge_target_row_present@7 as __sail_merge_target_row_present, __sail_src_value@9 as __sail_src_value, __sail_merge_source_row_present@10 as __sail_merge_source_row_present]
+                    MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
+                      RepartitionExec: partitioning=Hash([__sail_merge_target_row_id@6], 4), input_partitions=4
+                        HashJoinExec: mode=CollectLeft, join_type=Full, on=[(__sail_src_id@0, id@0)], projection=[id@3, value@4, __sail_file_path@5, __sail_file_row_index@6, __sail_iceberg_partition_spec_id@7, __sail_iceberg_partition@8, __sail_merge_target_row_id@9, __sail_merge_target_row_present@10, __sail_src_id@0, __sail_src_value@1, __sail_merge_source_row_present@2]
+                          ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_value, true as __sail_merge_source_row_present]
+                            DataSourceExec: partitions=1, partition_sizes=[1]
+                          RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3
+                            ProjectionExec: expr=[id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_file_row_index@3 as __sail_file_row_index, __sail_iceberg_partition_spec_id@4 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@5 as __sail_iceberg_partition, __sail_merge_target_row_id@6 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
+                              MonotonicIdExec: col=__sail_merge_target_row_id
+                                ProjectionExec: expr=[id@0 as id, value@1 as value, __sail_file_path@2 as __sail_file_path, __sail_file_row_index@5 as __sail_file_row_index, __sail_iceberg_partition_spec_id@3 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@4 as __sail_iceberg_partition]
+                                  IcebergMergeMetadataExec: data_file_column=__sail_file_path
+                                    DataSourceExec: file_groups={3 groups: [[/iceberg_merge_many_files_plan/data/part--00000000000000000000.parquet], [/iceberg_merge_many_files_plan/data/part--00000000000000000001.parquet], [/iceberg_merge_many_files_plan/data/part--00000000000000000002.parquet]]}, projection=[id, value, __sail_file_path, __sail_iceberg_partition_spec_id, __sail_iceberg_partition], file_type=parquet
+            ProjectionExec: expr=[NULL as id, NULL as value, NULL as __sail_file_path, NULL as __sail_file_row_index, NULL as __sail_iceberg_partition_spec_id, NULL as __sail_iceberg_partition, 9 as __sail_operation_type, 3 as __sail_merge_source_metric]
+              PlaceholderRowExec
+---
+name: "test_explain_shows_one_mergeonread_rowintent_writer"
+data:
+  |-
+    == Physical Plan ==
+    IcebergCommitExec(table_path=file:///iceberg_merge_plan/)
+      CoalescePartitionsExec
+        IcebergWriterExec(table_path=file:///iceberg_merge_plan/, merge_row_intents=true)
+          UnionExec
+            ProjectionExec: expr=[CASE WHEN __common_expr_1@0 THEN __sail_src_id@12 ELSE id@5 END as id, CASE WHEN __common_expr_2@1 THEN __sail_src_value@13 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@15 IS NULL AND flag@7 = expire) IS NOT DISTINCT FROM true THEN expired WHEN __common_expr_1@0 THEN __sail_src_value@13 ELSE value@6 END as value, CASE WHEN __common_expr_1@0 THEN __sail_src_flag@14 ELSE flag@7 END as flag, __sail_file_path@8 as __sail_file_path, __sail_file_row_index@9 as __sail_file_row_index, __sail_iceberg_partition_spec_id@10 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@11 as __sail_iceberg_partition, CASE WHEN __common_expr_4@3 THEN 3 WHEN __common_expr_2@1 THEN 6 WHEN (__common_expr_5@4 AND flag@7 = delete) IS NOT DISTINCT FROM true AND NOT __common_expr_2@1 THEN 5 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@15 IS NULL AND flag@7 = expire) IS NOT DISTINCT FROM true THEN 8 WHEN (__common_expr_3@2 AND __sail_merge_source_row_present@15 IS NULL AND flag@7 = purge) IS NOT DISTINCT FROM true AND NOT (__common_expr_3@2 AND __sail_merge_source_row_present@15 IS NULL AND flag@7 = expire) IS NOT DISTINCT FROM true THEN 7 ELSE 0 END as __sail_operation_type, NULL as __sail_merge_source_metric]
+              ProjectionExec: expr=[(__common_expr_6@0 AND __sail_merge_source_row_present@13 IS NOT NULL) IS NOT DISTINCT FROM true as __common_expr_1, (__common_expr_7@1 AND flag@5 = update) IS NOT DISTINCT FROM true as __common_expr_2, __common_expr_8@2 as __common_expr_3, __common_expr_6@0 as __common_expr_4, __common_expr_7@1 as __common_expr_5, id@3 as id, value@4 as value, flag@5 as flag, __sail_file_path@6 as __sail_file_path, __sail_file_row_index@7 as __sail_file_row_index, __sail_iceberg_partition_spec_id@8 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@9 as __sail_iceberg_partition, __sail_src_id@10 as __sail_src_id, __sail_src_value@11 as __sail_src_value, __sail_src_flag@12 as __sail_src_flag, __sail_merge_source_row_present@13 as __sail_merge_source_row_present]
+                ProjectionExec: expr=[__sail_merge_target_row_present@8 IS NULL as __common_expr_6, __common_expr_9@0 AND __sail_merge_source_row_present@12 IS NOT NULL as __common_expr_7, __common_expr_9@0 as __common_expr_8, id@1 as id, value@2 as value, flag@3 as flag, __sail_file_path@4 as __sail_file_path, __sail_file_row_index@5 as __sail_file_row_index, __sail_iceberg_partition_spec_id@6 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@7 as __sail_iceberg_partition, __sail_src_id@9 as __sail_src_id, __sail_src_value@10 as __sail_src_value, __sail_src_flag@11 as __sail_src_flag, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                  ProjectionExec: expr=[__sail_merge_target_row_present@7 IS NOT NULL as __common_expr_9, id@0 as id, value@1 as value, flag@2 as flag, __sail_file_path@3 as __sail_file_path, __sail_file_row_index@4 as __sail_file_row_index, __sail_iceberg_partition_spec_id@5 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@6 as __sail_iceberg_partition, __sail_merge_target_row_present@7 as __sail_merge_target_row_present, __sail_src_id@8 as __sail_src_id, __sail_src_value@9 as __sail_src_value, __sail_src_flag@10 as __sail_src_flag, __sail_merge_source_row_present@11 as __sail_merge_source_row_present]
+                    FilterExec: __sail_merge_target_row_present@7 IS NOT NULL OR __sail_merge_target_row_present@7 IS NULL AND __sail_merge_source_row_present@11 IS NOT NULL
+                      ProjectionExec: expr=[id@0 as id, value@1 as value, flag@2 as flag, __sail_file_path@3 as __sail_file_path, __sail_file_row_index@4 as __sail_file_row_index, __sail_iceberg_partition_spec_id@5 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@6 as __sail_iceberg_partition, __sail_merge_target_row_present@8 as __sail_merge_target_row_present, __sail_src_id@9 as __sail_src_id, __sail_src_value@10 as __sail_src_value, __sail_src_flag@11 as __sail_src_flag, __sail_merge_source_row_present@12 as __sail_merge_source_row_present]
+                        MergeCardinalityCheckExec: row_id=__sail_merge_target_row_id, target_present=__sail_merge_target_row_present, source_present=__sail_merge_source_row_present
+                          RepartitionExec: partitioning=Hash([__sail_merge_target_row_id@7], 4), input_partitions=4
+                            HashJoinExec: mode=CollectLeft, join_type=Full, on=[(__sail_src_id@0, id@0)], projection=[id@4, value@5, flag@6, __sail_file_path@7, __sail_file_row_index@8, __sail_iceberg_partition_spec_id@9, __sail_iceberg_partition@10, __sail_merge_target_row_id@11, __sail_merge_target_row_present@12, __sail_src_id@0, __sail_src_value@1, __sail_src_flag@2, __sail_merge_source_row_present@3]
+                              ProjectionExec: expr=[column1@0 as __sail_src_id, column2@1 as __sail_src_value, column3@2 as __sail_src_flag, true as __sail_merge_source_row_present]
+                                DataSourceExec: partitions=1, partition_sizes=[1]
+                              RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+                                ProjectionExec: expr=[id@0 as id, value@1 as value, flag@2 as flag, __sail_file_path@3 as __sail_file_path, __sail_file_row_index@4 as __sail_file_row_index, __sail_iceberg_partition_spec_id@5 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@6 as __sail_iceberg_partition, __sail_merge_target_row_id@7 as __sail_merge_target_row_id, true as __sail_merge_target_row_present]
+                                  MonotonicIdExec: col=__sail_merge_target_row_id
+                                    ProjectionExec: expr=[id@0 as id, value@1 as value, flag@2 as flag, __sail_file_path@3 as __sail_file_path, __sail_file_row_index@6 as __sail_file_row_index, __sail_iceberg_partition_spec_id@4 as __sail_iceberg_partition_spec_id, __sail_iceberg_partition@5 as __sail_iceberg_partition]
+                                      IcebergMergeMetadataExec: data_file_column=__sail_file_path
+                                        DataSourceExec: file_groups={1 group: [[/iceberg_merge_plan/data/part--00000000000000000000.parquet]]}, projection=[id, value, flag, __sail_file_path, __sail_iceberg_partition_spec_id, __sail_iceberg_partition], file_type=parquet
+            ProjectionExec: expr=[NULL as id, NULL as value, NULL as flag, NULL as __sail_file_path, NULL as __sail_file_row_index, NULL as __sail_iceberg_partition_spec_id, NULL as __sail_iceberg_partition, 9 as __sail_operation_type, 3 as __sail_merge_source_metric]
+              PlaceholderRowExec
+---
+name: "test_merge_writes_overwrite_metadata_with_data_and_positiondelete_manifests"
+data:
+  current-schema-id: 0
+  current-snapshot-id: 
+  default-sort-order-id: 0
+  default-spec-id: 0
+  format-version: 2
+  last-column-id: 3
+  last-partition-id: 999
+  last-sequence-number: 2
+  last-updated-ms: 
+  location: file:///iceberg_merge_metadata/
+  metadata-log:
+  - metadata-file: /iceberg_merge_metadata/metadata/v1.metadata.json
+    timestamp-ms: 
+  - metadata-file: /iceberg_merge_metadata/metadata/v2.metadata.json
+    timestamp-ms: 
+  partition-specs:
+  - fields: []
+    spec-id: 0
+  properties:
+    write.merge.mode: merge-on-read
+  refs:
+    main:
+      snapshot-id: 
+      type: branch
+  schemas:
+  - fields:
+    - id: 1
+      name: id
+      required: false
+      type: int
+    - id: 2
+      name: value
+      required: false
+      type: string
+    - id: 3
+      name: flag
+      required: false
+      type: string
+    schema-id: 0
+    type: struct
+  snapshot-log:
+  - snapshot-id: 
+    timestamp-ms: 
+  - snapshot-id: 
+    timestamp-ms: 
+  snapshots:
+  - manifest-list: file:///metadata/snap-.avro
+    schema-id: 0
+    sequence-number: 1
+    snapshot-id: 
+    summary:
+      added-data-files: '1'
+      added-records: '5'
+      operation: append
+      total-data-files: '1'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '5'
+    timestamp-ms: 
+  - manifest-list: file:///metadata/snap-.avro
+    parent-snapshot-id: 
+    schema-id: 0
+    sequence-number: 2
+    snapshot-id: 
+    summary:
+      added-data-files: '2'
+      added-delete-files: '2'
+      added-position-delete-files: '2'
+      added-position-deletes: '4'
+      added-records: '3'
+      operation: overwrite
+      total-data-files: '3'
+      total-delete-files: '2'
+      total-equality-deletes: '0'
+      total-position-deletes: '4'
+      total-records: '8'
+    timestamp-ms: 
+  sort-orders:
+  - fields: []
+    order-id: 0
+  table-uuid: 
+---
+name: "test_merge_writes_overwrite_metadata_with_data_and_positiondelete_manifests.1"
+data:
+  manifests:
+  - added-files-count: 1
+    added-rows-count: 5
+    added-snapshot-id: 
+    content: data
+    deleted-files-count: 0
+    deleted-rows-count: 0
+    existing-files-count: 0
+    existing-rows-count: 0
+    key-metadata: null
+    manifest-length: 
+    manifest-path: file:///metadata/manifest-.avro
+    min-sequence-number: 1
+    partition-spec-id: 0
+    partitions: null
+    sequence-number: 1
+  - added-files-count: 2
+    added-rows-count: 3
+    added-snapshot-id: 
+    content: data
+    deleted-files-count: 0
+    deleted-rows-count: 0
+    existing-files-count: 0
+    existing-rows-count: 0
+    key-metadata: null
+    manifest-length: 
+    manifest-path: file:///metadata/manifest-.avro
+    min-sequence-number: 2
+    partition-spec-id: 0
+    partitions: null
+    sequence-number: 2
+  - added-files-count: 2
+    added-rows-count: 4
+    added-snapshot-id: 
+    content: deletes
+    deleted-files-count: 0
+    deleted-rows-count: 0
+    existing-files-count: 0
+    existing-rows-count: 0
+    key-metadata: null
+    manifest-length: 
+    manifest-path: file:///metadata/manifest-.avro
+    min-sequence-number: 2
+    partition-spec-id: 0
+    partitions: null
+    sequence-number: 2
+---
+name: "test_merge_writes_overwrite_metadata_with_data_and_positiondelete_manifests.2"
+data:
+  added-data-files: '2'
+  added-delete-files: '2'
+  added-position-delete-files: '2'
+  added-position-deletes: '4'
+  added-records: '3'
+  operation: overwrite
+  total-data-files: '3'
+  total-delete-files: '2'
+  total-equality-deletes: '0'
+  total-position-deletes: '4'
+  total-records: '8'
+---
+name: "test_merge_writes_overwrite_metadata_with_data_and_positiondelete_manifests.3"
+data:
+  current-schema-id: 0
+  last-column-id: 3
+  schemas:
+  - fields:
+    - id: 1
+      name: id
+      required: false
+      type: int
+    - id: 2
+      name: value
+      required: false
+      type: string
+    - id: 3
+      name: flag
+      required: false
+      type: string
+    schema-id: 0
+    type: struct
diff --git a/python/pysail/tests/spark/iceberg/__snapshots__/features/partition_transforms.yaml b/python/pysail/tests/spark/iceberg/__snapshots__/features/partition_transforms.yaml
index 0c59e25b3..2d9107f25 100644
--- a/python/pysail/tests/spark/iceberg/__snapshots__/features/partition_transforms.yaml
+++ b/python/pysail/tests/spark/iceberg/__snapshots__/features/partition_transforms.yaml
@@ -52,7 +52,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '3'
+      added-records: '4'
       operation: append
+      total-data-files: '3'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -119,7 +126,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '4'
+      added-records: '4'
       operation: append
+      total-data-files: '4'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -193,7 +207,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '3'
+      added-records: '4'
       operation: append
+      total-data-files: '3'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -252,7 +273,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '3'
+      added-records: '4'
       operation: append
+      total-data-files: '3'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -311,7 +339,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '3'
+      added-records: '4'
       operation: append
+      total-data-files: '3'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -320,7 +355,14 @@ data:
 ---
 name: "test_month_transform_partitions_by_year_and_month.1"
 data:
+  added-data-files: '3'
+  added-records: '4'
   operation: append
+  total-data-files: '3'
+  total-delete-files: '0'
+  total-equality-deletes: '0'
+  total-position-deletes: '0'
+  total-records: '4'
 ---
 name: "test_truncate_on_string_column_groups_similar_values"
 data:
@@ -374,7 +416,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '2'
+      added-records: '4'
       operation: append
+      total-data-files: '2'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -444,7 +493,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '2'
+      added-records: '4'
       operation: append
+      total-data-files: '2'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '4'
     timestamp-ms: 
   sort-orders:
   - fields: []
diff --git a/python/pysail/tests/spark/iceberg/__snapshots__/features/variant.yaml b/python/pysail/tests/spark/iceberg/__snapshots__/features/variant.yaml
index 3ee23c945..4a0ab3828 100644
--- a/python/pysail/tests/spark/iceberg/__snapshots__/features/variant.yaml
+++ b/python/pysail/tests/spark/iceberg/__snapshots__/features/variant.yaml
@@ -8,7 +8,7 @@ data:
   default-spec-id: 0
   format-version: 2
   last-column-id: 1
-  last-partition-id: 0
+  last-partition-id: 999
   last-sequence-number: 1
   last-updated-ms: 
   location: file:///iceberg_variant_schema_evolution/
@@ -40,7 +40,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '1'
+      added-records: '1'
       operation: append
+      total-data-files: '1'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '1'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -55,7 +62,7 @@ data:
   default-spec-id: 0
   format-version: 3
   last-column-id: 2
-  last-partition-id: 0
+  last-partition-id: 999
   last-sequence-number: 2
   last-updated-ms: 
   location: file:///iceberg_variant_schema_evolution/
@@ -103,7 +110,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '1'
+      added-records: '1'
       operation: append
+      total-data-files: '1'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '1'
     timestamp-ms: 
   - added-rows: 2
     first-row-id: 0
@@ -113,7 +127,14 @@ data:
     sequence-number: 2
     snapshot-id: 
     summary:
+      added-data-files: '1'
+      added-records: '1'
       operation: append
+      total-data-files: '2'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '2'
     timestamp-ms: 
   sort-orders:
   - fields: []
@@ -164,7 +185,7 @@ data:
   default-spec-id: 0
   format-version: 3
   last-column-id: 2
-  last-partition-id: 0
+  last-partition-id: 999
   last-sequence-number: 1
   last-updated-ms: 
   location: file:///iceberg_variant/
@@ -203,7 +224,14 @@ data:
     sequence-number: 1
     snapshot-id: 
     summary:
+      added-data-files: '1'
+      added-records: '1'
       operation: append
+      total-data-files: '1'
+      total-delete-files: '0'
+      total-equality-deletes: '0'
+      total-position-deletes: '0'
+      total-records: '1'
     timestamp-ms: 
   sort-orders:
   - fields: []
diff --git a/python/pysail/tests/spark/iceberg/features/alter_table.feature b/python/pysail/tests/spark/iceberg/features/alter_table.feature
index 99961ceac..f1ea0f7de 100644
--- a/python/pysail/tests/spark/iceberg/features/alter_table.feature
+++ b/python/pysail/tests/spark/iceberg/features/alter_table.feature
@@ -36,6 +36,46 @@ Feature: Iceberg ALTER TABLE SET/UNSET TBLPROPERTIES
           📄 snap-*.avro
         """
 
+    Scenario: Absolute write data path stores files outside the table location
+      Given variable location for temporary directory iceberg_absolute_write_data_path_table
+      Given variable data_location for temporary directory iceberg_absolute_write_data_path_data
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_absolute_write_data_path_test
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_absolute_write_data_path_test (
+          id INT,
+          value STRING
+        )
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES ('write.data.path' = '{{ data_location.file_uri }}')
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_absolute_write_data_path_test VALUES (1, 'v0')
+        """
+      Then file tree in location matches
+        """
+        📂 metadata
+          📄 *.metadata.json
+          📄 *.metadata.json
+          📄 snap-*.avro
+        """
+      Then file tree in data_location matches
+        """
+        📄 *.parquet
+        """
+      When query
+        """
+        SELECT * FROM iceberg_absolute_write_data_path_test
+        """
+      Then query result
+        | id | value |
+        | 1  | v0    |
+
     Scenario: CREATE TABLE format-version selects metadata version without persisting reserved properties
       Given variable location for temporary directory iceberg_create_table_format_version
       Given final statement
diff --git a/python/pysail/tests/spark/iceberg/features/merge.feature b/python/pysail/tests/spark/iceberg/features/merge.feature
new file mode 100644
index 000000000..460487fd5
--- /dev/null
+++ b/python/pysail/tests/spark/iceberg/features/merge.feature
@@ -0,0 +1,596 @@
+Feature: Iceberg MERGE
+
+  Rule: MERGE conditions must be deterministic
+
+    Scenario: A non-deterministic matched condition is rejected before writing
+      Given variable location for temporary directory iceberg_merge_nondeterministic
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_nondeterministic
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_nondeterministic (id INT, value STRING)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_nondeterministic VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_nondeterministic_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      When query
+        """
+        MERGE INTO iceberg_merge_nondeterministic AS t
+        USING iceberg_merge_nondeterministic_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND rand() < 0.5 THEN UPDATE SET value = s.value
+        """
+      Then query error Non-deterministic expressions are not allowed in MERGE conditions
+      When query
+        """
+        SELECT id, value FROM iceberg_merge_nondeterministic
+        """
+      Then query result
+        | id | value |
+        | 1  | old   |
+
+  Rule: MERGE assignments follow target schema semantics
+
+    Scenario: A legal target column does not collide with generated source aliases
+      Given variable location for temporary directory iceberg_merge_source_alias
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_source_alias
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_source_alias (`__sail_src_id` INT, value STRING)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_source_alias VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_source_alias_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      Given statement
+        """
+        MERGE INTO iceberg_merge_source_alias AS t
+        USING iceberg_merge_source_alias_source AS s
+        ON t.`__sail_src_id` = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      When query
+        """
+        SELECT `__sail_src_id`, value FROM iceberg_merge_source_alias
+        """
+      Then query result
+        | __sail_src_id | value |
+        | 1             | new   |
+
+    Scenario: Star actions reject source columns missing from the target schema
+      Given variable location for temporary directory iceberg_merge_star_missing
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_star_missing
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_star_missing (id INT, value STRING, keep STRING)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_star_missing VALUES (1, 'old', 'preserved')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_star_missing_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      When query
+        """
+        MERGE INTO iceberg_merge_star_missing AS t
+        USING iceberg_merge_star_missing_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET *
+        WHEN NOT MATCHED THEN INSERT *
+        """
+      Then query error Cannot resolve source column `keep` for MERGE \* action without schema evolution
+
+    Scenario: Assignments cast to target types and reject overflow
+      Given config spark.sql.ansi.enabled = true
+      Given variable location for temporary directory iceberg_merge_assignment_cast
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_assignment_cast
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_assignment_cast (id INT, value INT)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_assignment_cast VALUES (1, 10)
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_assignment_cast_source AS
+        SELECT 1 AS id, CAST(20 AS BIGINT) AS value
+        """
+      Given statement
+        """
+        MERGE INTO iceberg_merge_assignment_cast AS t
+        USING iceberg_merge_assignment_cast_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      When query
+        """
+        SELECT id, value FROM iceberg_merge_assignment_cast
+        """
+      Then query result
+        | id | value |
+        | 1  | 20    |
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_assignment_cast_source AS
+        SELECT 1 AS id, CAST(2147483648 AS BIGINT) AS value
+        """
+      When query
+        """
+        MERGE INTO iceberg_merge_assignment_cast AS t
+        USING iceberg_merge_assignment_cast_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      Then query error (?i).*(cast|overflow).*
+      When query
+        """
+        SELECT id, value FROM iceberg_merge_assignment_cast
+        """
+      Then query result
+        | id | value |
+        | 1  | 20    |
+
+    Scenario: Duplicate target assignments are rejected
+      Given variable location for temporary directory iceberg_merge_duplicate_assignment
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_duplicate_assignment
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_duplicate_assignment (id INT, value STRING)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_duplicate_assignment VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_duplicate_assignment_source AS
+        SELECT 1 AS id
+        """
+      When query
+        """
+        MERGE INTO iceberg_merge_duplicate_assignment AS t
+        USING iceberg_merge_duplicate_assignment_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = 'first', value = 'second'
+        """
+      Then query error Multiple assignments for MERGE target column
+
+    Scenario: Star matching honors case-sensitive resolution
+      Given config spark.sql.caseSensitive = true
+      Given variable location for temporary directory iceberg_merge_case_sensitive
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_case_sensitive
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_case_sensitive (`A` INT)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_case_sensitive VALUES (1)
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_case_sensitive_source AS
+        SELECT 1 AS a
+        """
+      When query
+        """
+        MERGE INTO iceberg_merge_case_sensitive AS t
+        USING iceberg_merge_case_sensitive_source AS s
+        ON t.`A` = s.a
+        WHEN MATCHED THEN UPDATE SET *
+        """
+      Then query error Cannot resolve source column `A` for MERGE \* action without schema evolution
+
+    Scenario: Unqualified MERGE columns honor case-sensitive resolution
+      Given config spark.sql.caseSensitive = true
+      Given variable location for temporary directory iceberg_merge_unqualified_case_sensitive
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_unqualified_case_sensitive
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_unqualified_case_sensitive (`A` INT)
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_unqualified_case_sensitive VALUES (1)
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_unqualified_case_sensitive_source AS
+        SELECT 2 AS a
+        """
+      Given statement
+        """
+        MERGE INTO iceberg_merge_unqualified_case_sensitive AS t
+        USING iceberg_merge_unqualified_case_sensitive_source AS s
+        ON `A` = a
+        WHEN MATCHED THEN UPDATE SET `A` = s.a
+        WHEN NOT MATCHED THEN INSERT (`A`) VALUES (s.a)
+        """
+      When query
+        """
+        SELECT `A` FROM iceberg_merge_unqualified_case_sensitive ORDER BY `A`
+        """
+      Then query result ordered
+        | A |
+        | 1 |
+        | 2 |
+
+  Rule: Merge-on-read execution plans and metadata
+
+    Scenario: EXPLAIN shows one merge-on-read row-intent writer
+      Given variable location for temporary directory iceberg_merge_plan
+      Given final statement
+        """
+        DROP TABLE IF EXISTS merge_plan_table
+        """
+      Given statement template
+        """
+        CREATE TABLE merge_plan_table (
+          id INT,
+          value STRING,
+          flag STRING
+        )
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO merge_plan_table
+        SELECT * FROM VALUES
+          (1, 'old', 'keep'),
+          (2, 'old', 'update'),
+          (3, 'old', 'delete'),
+          (5, 'old', 'expire'),
+          (6, 'old', 'purge')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW merge_plan_source AS
+        SELECT * FROM VALUES
+          (2, 'new', 'insert'),
+          (3, 'ignored', 'delete'),
+          (4, 'ins', 'insert')
+        AS src(id, value, flag)
+        """
+      When query
+        """
+        EXPLAIN MERGE INTO merge_plan_table AS t
+        USING merge_plan_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND t.flag = 'update' THEN
+          UPDATE SET value = s.value
+        WHEN MATCHED AND t.flag = 'delete' THEN
+          DELETE
+        WHEN NOT MATCHED THEN
+          INSERT (id, value, flag) VALUES (s.id, s.value, s.flag)
+        WHEN NOT MATCHED BY SOURCE AND t.flag = 'expire' THEN
+          UPDATE SET value = 'expired'
+        WHEN NOT MATCHED BY SOURCE AND t.flag = 'purge' THEN
+          DELETE
+        """
+      Then query plan matches snapshot
+
+    Scenario: EXPLAIN keeps merge metadata scans constant across target files
+      Given variable location for temporary directory iceberg_merge_many_files_plan
+      Given final statement
+        """
+        DROP TABLE IF EXISTS merge_many_files_plan_table
+        """
+      Given statement template
+        """
+        CREATE TABLE merge_many_files_plan_table (
+          id INT,
+          value STRING
+        )
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO merge_many_files_plan_table VALUES (1, 'one')
+        """
+      Given statement
+        """
+        INSERT INTO merge_many_files_plan_table VALUES (2, 'two')
+        """
+      Given statement
+        """
+        INSERT INTO merge_many_files_plan_table VALUES (3, 'three')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW merge_many_files_plan_source AS
+        SELECT * FROM VALUES
+          (1, 'updated-one'),
+          (2, 'updated-two'),
+          (3, 'updated-three')
+        AS source(id, value)
+        """
+      When query
+        """
+        EXPLAIN MERGE INTO merge_many_files_plan_table AS t
+        USING merge_many_files_plan_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      Then query plan matches snapshot
+      Given statement
+        """
+        MERGE INTO merge_many_files_plan_table AS t
+        USING merge_many_files_plan_source AS s
+        ON t.id = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      When query
+        """
+        SELECT id, value FROM merge_many_files_plan_table ORDER BY id
+        """
+      Then query result ordered
+        | id | value         |
+        | 1  | updated-one   |
+        | 2  | updated-two   |
+        | 3  | updated-three |
+
+    Scenario: MERGE writes overwrite metadata with data and position-delete manifests
+      Given variable location for temporary directory iceberg_merge_metadata
+      Given final statement
+        """
+        DROP TABLE IF EXISTS merge_metadata_table
+        """
+      Given statement template
+        """
+        CREATE TABLE merge_metadata_table (
+          id INT,
+          value STRING,
+          flag STRING
+        )
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO merge_metadata_table
+        SELECT * FROM VALUES
+          (1, 'old', 'keep'),
+          (2, 'old', 'update'),
+          (3, 'old', 'delete'),
+          (5, 'old', 'expire'),
+          (6, 'old', 'purge')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW merge_metadata_source AS
+        SELECT * FROM VALUES
+          (2, 'new', 'insert'),
+          (3, 'ignored', 'delete'),
+          (4, 'ins', 'insert')
+        AS src(id, value, flag)
+        """
+      Given statement
+        """
+        MERGE INTO merge_metadata_table AS t
+        USING merge_metadata_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND t.flag = 'update' THEN
+          UPDATE SET value = s.value
+        WHEN MATCHED AND t.flag = 'delete' THEN
+          DELETE
+        WHEN NOT MATCHED THEN
+          INSERT (id, value, flag) VALUES (s.id, s.value, s.flag)
+        WHEN NOT MATCHED BY SOURCE AND t.flag = 'expire' THEN
+          UPDATE SET value = 'expired'
+        WHEN NOT MATCHED BY SOURCE AND t.flag = 'purge' THEN
+          DELETE
+        """
+      Then iceberg metadata matches snapshot
+      Then iceberg current manifest list matches snapshot
+      Then iceberg current snapshot summary matches snapshot
+      Then iceberg schema history matches snapshot
+      Then iceberg snapshot count is 2
+      When query
+        """
+        SELECT id, value, flag FROM merge_metadata_table ORDER BY id
+        """
+      Then query result ordered
+        | id | value   | flag   |
+        | 1  | old     | keep   |
+        | 2  | new     | update |
+        | 4  | ins     | insert |
+        | 5  | expired | expire |
+
+  Rule: WHEN clauses use first-match semantics
+
+    Scenario: Overlapping matched and target-only clauses apply only their first action
+      Given variable location for temporary directory iceberg_merge_first_match
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_first_match
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_first_match (
+          id INT,
+          left_value STRING,
+          right_value STRING,
+          kind STRING
+        )
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_first_match VALUES
+          (1, 'old-left', 'old-right', 'delete-update'),
+          (2, 'old-left', 'old-right', 'partial-update'),
+          (3, 'old-left', 'old-right', 'source-update-delete'),
+          (4, 'old-left', 'old-right', 'source-delete-update')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_first_match_source AS
+        SELECT * FROM VALUES
+          (1, 'new-left', 'new-right', 'delete-update'),
+          (2, 'new-left', 'new-right', 'partial-update')
+        AS src(id, left_value, right_value, kind)
+        """
+      Given statement
+        """
+        MERGE INTO iceberg_merge_first_match AS t
+        USING iceberg_merge_first_match_source AS s
+        ON t.id = s.id
+        WHEN MATCHED AND s.kind = 'delete-update' THEN DELETE
+        WHEN MATCHED AND s.kind IN ('delete-update', 'partial-update') THEN
+          UPDATE SET left_value = s.left_value
+        WHEN MATCHED AND s.kind = 'partial-update' THEN
+          UPDATE SET right_value = s.right_value
+        WHEN MATCHED THEN DELETE
+        WHEN NOT MATCHED BY SOURCE AND t.kind = 'source-delete-update' THEN DELETE
+        WHEN NOT MATCHED BY SOURCE AND t.kind IN ('source-update-delete', 'source-delete-update') THEN
+          UPDATE SET left_value = 'source-left'
+        WHEN NOT MATCHED BY SOURCE AND t.kind = 'source-update-delete' THEN
+          UPDATE SET right_value = 'source-right'
+        WHEN NOT MATCHED BY SOURCE THEN DELETE
+        """
+      When query
+        """
+        SELECT id, left_value, right_value, kind
+        FROM iceberg_merge_first_match
+        ORDER BY id
+        """
+      Then query result ordered
+        | id | left_value  | right_value | kind                 |
+        | 2  | new-left    | old-right   | partial-update       |
+        | 3  | source-left | old-right   | source-update-delete |
+
+  Rule: Internal MERGE columns cannot shadow table data
+
+    Scenario: A target column using an internal operation name is rejected clearly
+      Given variable location for temporary directory iceberg_merge_internal_column
+      Given final statement
+        """
+        DROP TABLE IF EXISTS iceberg_merge_internal_column
+        """
+      Given statement template
+        """
+        CREATE TABLE iceberg_merge_internal_column (
+          `__sail_operation_type` INT,
+          value STRING
+        )
+        USING iceberg
+        LOCATION {{ location.uri }}
+        TBLPROPERTIES (
+          'format-version' = '2',
+          'write.merge.mode' = 'merge-on-read'
+        )
+        """
+      Given statement
+        """
+        INSERT INTO iceberg_merge_internal_column VALUES (1, 'old')
+        """
+      Given statement
+        """
+        CREATE OR REPLACE TEMP VIEW iceberg_merge_internal_column_source AS
+        SELECT 1 AS id, 'new' AS value
+        """
+      When query
+        """
+        MERGE INTO iceberg_merge_internal_column AS t
+        USING iceberg_merge_internal_column_source AS s
+        ON t.`__sail_operation_type` = s.id
+        WHEN MATCHED THEN UPDATE SET value = s.value
+        """
+      Then query error reserved internal MERGE column
diff --git a/python/pysail/tests/spark/iceberg/test_iceberg_equality_delete.py b/python/pysail/tests/spark/iceberg/test_iceberg_equality_delete.py
new file mode 100644
index 000000000..3a6b18b12
--- /dev/null
+++ b/python/pysail/tests/spark/iceberg/test_iceberg_equality_delete.py
@@ -0,0 +1,546 @@
+import time
+import uuid
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import url2pathname
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+from pyiceberg.io.pyarrow import PyArrowFileIO
+from pyiceberg.manifest import (
+    DataFile,
+    DataFileContent,
+    FileFormat,
+    ManifestContent,
+    ManifestEntry,
+    ManifestEntryStatus,
+    ManifestWriterV2,
+    read_manifest_list,
+    write_manifest_list,
+)
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.transforms import IdentityTransform
+from pyiceberg.typedef import Record
+from pyiceberg.types import LongType, NestedField, StringType
+
+from pysail.testing.spark.steps.iceberg import (
+    _current_manifest_list,
+    _current_snapshot,
+    _find_latest_metadata,
+    _latest_metadata_path,
+    _metadata_file_version,
+    _pyarrow_input_file,
+    _write_metadata_file,
+)
+from pysail.testing.spark.utils.sql import escape_sql_string_literal
+from pysail.tests.spark.iceberg.utils import create_sql_catalog
+
+UNPARTITIONED_LAST_PARTITION_ID = 999
+
+
+def _uri_sql(path: Path) -> str:
+    return escape_sql_string_literal(path.as_uri())
+
+
+def _drop_table(spark, name: str) -> None:
+    spark.sql(f"DROP TABLE IF EXISTS {name}")
+
+
+def _parquet_file_paths(table_path: Path) -> set[Path]:
+    return {path.relative_to(table_path) for path in table_path.rglob("*.parquet")}
+
+
+class _EqualityDeleteManifestWriter(ManifestWriterV2):
+    def content(self) -> ManifestContent:
+        return ManifestContent.DELETES
+
+    @property
+    def _meta(self) -> dict[str, str]:
+        meta = dict(super()._meta)
+        meta["content"] = "deletes"
+        return meta
+
+
+def _local_table_path(location: str) -> Path:
+    parsed = urlparse(location)
+    return Path(url2pathname(f"//{parsed.netloc}{parsed.path}" if parsed.netloc else parsed.path))
+
+
+def _new_snapshot_id() -> int:
+    return uuid.uuid4().int & ((1 << 63) - 1)
+
+
+def _append_equality_delete_snapshot(
+    table,
+    delete_rows: pa.Table,
+    equality_ids: list[int],
+    *,
+    partition: Record | None = None,
+) -> Path:
+    table_path = _local_table_path(table.location())
+    metadata_dir = table_path / "metadata"
+    data_dir = table_path / "data"
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    delete_file_path = data_dir / f"equality-delete-{uuid.uuid4()}.parquet"
+    pq.write_table(delete_rows, delete_file_path)
+
+    metadata = _find_latest_metadata(table_path)
+    parent_snapshot = _current_snapshot(metadata)
+    snapshot_id = _new_snapshot_id()
+    sequence_number = metadata.get("last-sequence-number", 0) + 1
+
+    io = PyArrowFileIO()
+    delete_data_file = DataFile.from_args(
+        content=DataFileContent.EQUALITY_DELETES,
+        file_path=delete_file_path.as_uri(),
+        file_format=FileFormat.PARQUET,
+        partition=partition or Record(),
+        record_count=delete_rows.num_rows,
+        file_size_in_bytes=delete_file_path.stat().st_size,
+        equality_ids=equality_ids,
+    )
+
+    manifest_path = metadata_dir / f"manifest-{uuid.uuid4()}.avro"
+    with _EqualityDeleteManifestWriter(
+        table.spec(),
+        table.schema(),
+        io.new_output(manifest_path.as_uri()),
+        snapshot_id,
+        "null",
+    ) as writer:
+        writer.add_entry(
+            ManifestEntry.from_args(
+                status=ManifestEntryStatus.ADDED,
+                snapshot_id=snapshot_id,
+                data_file=delete_data_file,
+            )
+        )
+        delete_manifest = writer.to_manifest_file()
+
+    existing_manifests = list(read_manifest_list(_pyarrow_input_file(io, parent_snapshot["manifest-list"])))
+    manifest_list_path = metadata_dir / f"snap-{snapshot_id}-0-{uuid.uuid4()}.avro"
+    with write_manifest_list(
+        2,
+        io.new_output(manifest_list_path.as_uri()),
+        snapshot_id,
+        parent_snapshot["snapshot-id"],
+        sequence_number,
+        "null",
+    ) as writer:
+        writer.add_manifests([*existing_manifests, delete_manifest])
+
+    latest_metadata_path = _latest_metadata_path(table_path)
+    latest_metadata_version = _metadata_file_version(latest_metadata_path)
+    assert latest_metadata_version is not None
+
+    previous_updated_ms = metadata["last-updated-ms"]
+    updated_ms = int(time.time() * 1000)
+    metadata["current-snapshot-id"] = snapshot_id
+    metadata["last-sequence-number"] = sequence_number
+    metadata["last-updated-ms"] = updated_ms
+    metadata.setdefault("snapshots", []).append(
+        {
+            "snapshot-id": snapshot_id,
+            "parent-snapshot-id": parent_snapshot["snapshot-id"],
+            "sequence-number": sequence_number,
+            "timestamp-ms": updated_ms,
+            "manifest-list": manifest_list_path.as_uri(),
+            "summary": {
+                "operation": "delete",
+                "added-delete-files": "1",
+                "added-equality-delete-files": "1",
+                "added-equality-deletes": str(delete_rows.num_rows),
+            },
+            "schema-id": metadata.get("current-schema-id", 0),
+        }
+    )
+    metadata.setdefault("snapshot-log", []).append({"snapshot-id": snapshot_id, "timestamp-ms": updated_ms})
+    metadata.setdefault("metadata-log", []).append(
+        {"metadata-file": latest_metadata_path.as_uri(), "timestamp-ms": previous_updated_ms}
+    )
+    metadata.setdefault("refs", {})["main"] = {"snapshot-id": snapshot_id, "type": "branch"}
+
+    new_metadata_path = metadata_dir / f"{latest_metadata_version + 1:05d}-{uuid.uuid4()}.metadata.json"
+    _write_metadata_file(new_metadata_path, metadata)
+    return table_path
+
+
+def _current_delete_entries(table_path: Path) -> list[ManifestEntry]:
+    metadata = _find_latest_metadata(table_path)
+    snapshot = _current_snapshot(metadata)
+    io = PyArrowFileIO()
+    entries = []
+    for manifest in read_manifest_list(_pyarrow_input_file(io, snapshot["manifest-list"])):
+        if manifest.content == ManifestContent.DELETES:
+            entries.extend(manifest.fetch_manifest_entry(io))
+    return entries
+
+
+def _delete_manifest_count(table_path: Path, key: str) -> int:
+    manifests = _current_manifest_list(_find_latest_metadata(table_path))["manifests"]
+    return sum(manifest.get(key) or 0 for manifest in manifests if manifest.get("content") == "deletes")
+
+
+def _assert_current_snapshot_metadata(
+    table_path: Path,
+    metadata: dict,
+    *,
+    expected_metadata_version: int,
+    expected_snapshot_count: int,
+) -> dict:
+    snapshot = _current_snapshot(metadata)
+    assert metadata["current-snapshot-id"] == snapshot["snapshot-id"]
+    assert metadata["last-sequence-number"] == snapshot["sequence-number"]
+    assert metadata["last-partition-id"] == UNPARTITIONED_LAST_PARTITION_ID
+    assert metadata["refs"]["main"]["snapshot-id"] == snapshot["snapshot-id"]
+    assert metadata["snapshot-log"][-1]["snapshot-id"] == snapshot["snapshot-id"]
+    assert len(metadata["snapshots"]) == expected_snapshot_count
+    assert _metadata_file_version(_latest_metadata_path(table_path)) == expected_metadata_version
+    assert metadata["metadata-log"][-1]["metadata-file"].endswith(
+        f"/metadata/v{expected_metadata_version - 1}.metadata.json"
+    )
+    return snapshot
+
+
+def test_iceberg_sql_delete_writes_equality_delete_file_and_filters_rows(spark, tmp_path):
+    table_name = "iceberg_sql_equality_delete"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id BIGINT,
+              name STRING,
+              flag STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.delete.mode' = 'merge-on-read',
+              'write.data.path' = 'custom_data'
+            )
+            """
+        )
+        spark.sql(
+            """
+            INSERT INTO iceberg_sql_equality_delete
+            SELECT * FROM VALUES
+              (1, 'keep-1', 'keep'),
+              (2, 'drop-2', 'drop'),
+              (3, 'keep-3', 'keep')
+            """
+        )
+
+        spark.sql("DELETE FROM iceberg_sql_equality_delete WHERE flag = 'drop'").collect()
+
+        rows = [
+            tuple(row)
+            for row in spark.sql("SELECT id, name, flag FROM iceberg_sql_equality_delete ORDER BY id").collect()
+        ]
+        assert rows == [(1, "keep-1", "keep"), (3, "keep-3", "keep")]
+
+        metadata = _find_latest_metadata(table_path)
+        snapshot = _assert_current_snapshot_metadata(
+            table_path,
+            metadata,
+            expected_metadata_version=3,
+            expected_snapshot_count=2,
+        )
+        summary = snapshot["summary"]
+        assert summary["operation"] == "delete"
+        assert summary["added-delete-files"] == "1"
+        assert summary["added-equality-delete-files"] == "1"
+        assert summary["added-equality-deletes"] == "1"
+        assert "deleted-records" not in summary
+        assert "added-position-delete-files" not in summary
+        assert summary["total-data-files"] == "1"
+        assert summary["total-delete-files"] == "1"
+        assert summary["total-records"] == "3"
+
+        manifests = _current_manifest_list(metadata)["manifests"]
+        data_manifests = [manifest for manifest in manifests if manifest.get("content") == "data"]
+        delete_manifests = [manifest for manifest in manifests if manifest.get("content") == "deletes"]
+        assert data_manifests
+        assert all(manifest["sequence-number"] < snapshot["sequence-number"] for manifest in data_manifests)
+        assert len(delete_manifests) == 1
+        assert delete_manifests[0]["sequence-number"] == snapshot["sequence-number"]
+        assert delete_manifests[0]["added-files-count"] == 1
+        assert delete_manifests[0]["added-rows-count"] == 1
+
+        assert _delete_manifest_count(table_path, "added-files-count") == 1
+        assert _delete_manifest_count(table_path, "added-rows-count") == 1
+
+        entries = _current_delete_entries(table_path)
+        assert len(entries) == 1
+        delete_file = entries[0].data_file
+        assert delete_file.content == DataFileContent.EQUALITY_DELETES
+        assert delete_file.equality_ids == [1, 2, 3]
+        assert getattr(delete_file, "referenced_data_file", None) is None
+        assert delete_file.file_path.startswith(f"{(table_path / 'custom_data').as_uri()}/equality-delete-")
+
+        delete_rows = pq.read_table(_local_table_path(delete_file.file_path)).to_pylist()
+        assert delete_rows == [{"id": 2, "name": "drop-2", "flag": "drop"}]
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_sql_delete_rejects_partitioned_equality_delete_without_metadata_commit(spark, tmp_path):
+    table_name = "iceberg_sql_equality_delete_partitioned_reject"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id BIGINT,
+              flag STRING,
+              part STRING
+            )
+            USING iceberg
+            PARTITIONED BY (part)
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.delete.mode' = 'merge-on-read'
+            )
+            """
+        )
+        empty_table_metadata_path = _latest_metadata_path(table_path)
+        with pytest.raises(Exception, match="partitioned tables are not supported"):
+            spark.sql("DELETE FROM iceberg_sql_equality_delete_partitioned_reject WHERE flag = 'drop'").collect()
+        assert _latest_metadata_path(table_path) == empty_table_metadata_path
+
+        spark.sql(
+            """
+            INSERT INTO iceberg_sql_equality_delete_partitioned_reject
+            SELECT * FROM VALUES
+              (1, 'keep', 'A'),
+              (2, 'drop', 'A'),
+              (3, 'keep', 'B')
+            """
+        )
+        before_metadata_path = _latest_metadata_path(table_path)
+
+        with pytest.raises(Exception, match="partitioned tables are not supported"):
+            spark.sql("DELETE FROM iceberg_sql_equality_delete_partitioned_reject WHERE flag = 'missing'").collect()
+        assert _latest_metadata_path(table_path) == before_metadata_path
+
+        with pytest.raises(Exception, match="partitioned tables are not supported"):
+            spark.sql("DELETE FROM iceberg_sql_equality_delete_partitioned_reject WHERE flag = 'drop'").collect()
+
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        metadata = _find_latest_metadata(table_path)
+        assert len(metadata["snapshots"]) == 1
+        rows = [
+            tuple(row)
+            for row in spark.sql(
+                "SELECT id, flag, part FROM iceberg_sql_equality_delete_partitioned_reject ORDER BY id"
+            ).collect()
+        ]
+        assert rows == [
+            (1, "keep", "A"),
+            (2, "drop", "A"),
+            (3, "keep", "B"),
+        ]
+    finally:
+        _drop_table(spark, table_name)
+
+
+@pytest.mark.parametrize("delete_mode", [None, "copy-on-write"], ids=["default", "explicit-cow"])
+def test_iceberg_sql_delete_rejects_copy_on_write_before_scanning_empty_table(spark, tmp_path, delete_mode):
+    table_name = "iceberg_delete_copy_on_write_reject"
+    table_path = tmp_path / table_name
+    mode_property = "" if delete_mode is None else f", 'write.delete.mode' = '{delete_mode}'"
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id BIGINT,
+              name STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES ('format-version' = '2'{mode_property})
+            """
+        )
+        before_metadata_path = _latest_metadata_path(table_path)
+        before_parquet_files = _parquet_file_paths(table_path)
+
+        with pytest.raises(Exception, match=r"write\.delete\.mode=copy-on-write|copy-on-write.*not supported"):
+            spark.sql("DELETE FROM iceberg_delete_copy_on_write_reject WHERE id = 1").collect()
+
+        assert spark.sql("SELECT * FROM iceberg_delete_copy_on_write_reject").collect() == []
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        assert _parquet_file_paths(table_path) == before_parquet_files
+    finally:
+        _drop_table(spark, table_name)
+
+
+@pytest.mark.parametrize("column_type", ["FLOAT", "DOUBLE"])
+def test_iceberg_sql_delete_rejects_floating_equality_keys_without_file_side_effects(spark, tmp_path, column_type):
+    table_name = "iceberg_delete_floating_key_reject"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id BIGINT,
+              measurement {column_type}
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.delete.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_delete_floating_key_reject VALUES (1, 1.25)")
+        before_metadata_path = _latest_metadata_path(table_path)
+        before_parquet_files = _parquet_file_paths(table_path)
+
+        with pytest.raises(Exception, match=rf"identifier-field-invalid type {column_type.lower()}"):
+            spark.sql("DELETE FROM iceberg_delete_floating_key_reject WHERE id = 1").collect()
+
+        assert [row.id for row in spark.sql("SELECT id FROM iceberg_delete_floating_key_reject").collect()] == [1]
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        assert _parquet_file_paths(table_path) == before_parquet_files
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_unpartitioned_equality_delete_filters_matching_rows_and_records_manifest(spark, tmp_path):
+    catalog = create_sql_catalog(tmp_path)
+    identifier = "default.equality_delete_global"
+    table = catalog.create_table(
+        identifier=identifier,
+        schema=Schema(
+            NestedField(1, "id", LongType(), required=False),
+            NestedField(2, "name", StringType(), required=False),
+        ),
+        properties={"format-version": "2"},
+    )
+    try:
+        table.append(pa.table({"id": [1, 2, 2, 3], "name": ["keep-1", "drop-a", "drop-b", "keep-3"]}))
+        table_path = _append_equality_delete_snapshot(table, pa.table({"id": [2]}), [1])
+
+        rows = [
+            tuple(row)
+            for row in spark.read.format("iceberg").load(table.location()).select("id", "name").orderBy("id").collect()
+        ]
+        assert rows == [(1, "keep-1"), (3, "keep-3")]
+
+        assert _delete_manifest_count(table_path, "added-files-count") == 1
+        assert _delete_manifest_count(table_path, "added-rows-count") == 1
+
+        entries = _current_delete_entries(table_path)
+        assert len(entries) == 1
+        assert entries[0].data_file.content == DataFileContent.EQUALITY_DELETES
+        assert entries[0].data_file.equality_ids == [1]
+        assert entries[0].sequence_number == _current_snapshot(_find_latest_metadata(table_path))["sequence-number"]
+    finally:
+        catalog.drop_table(identifier)
+
+
+def test_iceberg_equality_delete_matches_multiple_columns_with_nulls(spark, tmp_path):
+    catalog = create_sql_catalog(tmp_path)
+    identifier = "default.equality_delete_nulls"
+    table = catalog.create_table(
+        identifier=identifier,
+        schema=Schema(
+            NestedField(1, "id", LongType(), required=False),
+            NestedField(2, "category", StringType(), required=False),
+            NestedField(3, "marker", StringType(), required=False),
+        ),
+        properties={"format-version": "2"},
+    )
+    try:
+        table.append(
+            pa.table(
+                {
+                    "id": [1, 2, 3, 4, 5],
+                    "category": ["keep", "drop", "drop", None, None],
+                    "marker": ["x", None, "x", "x", None],
+                }
+            )
+        )
+        delete_rows = pa.table(
+            {
+                "category": pa.array(["drop", None], type=pa.string()),
+                "marker": pa.array([None, None], type=pa.string()),
+            }
+        )
+        _append_equality_delete_snapshot(table, delete_rows, [2, 3])
+
+        rows = [
+            tuple(row)
+            for row in (
+                spark.read.format("iceberg")
+                .load(table.location())
+                .select("id", "category", "marker")
+                .orderBy("id")
+                .collect()
+            )
+        ]
+        assert rows == [
+            (1, "keep", "x"),
+            (3, "drop", "x"),
+            (4, None, "x"),
+        ]
+    finally:
+        catalog.drop_table(identifier)
+
+
+def test_iceberg_partitioned_equality_delete_only_applies_within_delete_partition(spark, tmp_path):
+    catalog = create_sql_catalog(tmp_path)
+    identifier = "default.equality_delete_partitioned"
+    table = catalog.create_table(
+        identifier=identifier,
+        schema=Schema(
+            NestedField(1, "id", LongType(), required=False),
+            NestedField(2, "payload", StringType(), required=False),
+            NestedField(3, "part", StringType(), required=False),
+        ),
+        partition_spec=PartitionSpec(PartitionField(3, 1000, IdentityTransform(), "part")),
+        properties={"format-version": "2"},
+    )
+    try:
+        table.append(
+            pa.table(
+                {
+                    "id": [2, 2, 3],
+                    "payload": ["drop-a", "keep-b", "keep-a"],
+                    "part": ["A", "B", "A"],
+                }
+            )
+        )
+        table_path = _append_equality_delete_snapshot(table, pa.table({"id": [2]}), [1], partition=Record("A"))
+
+        rows = [
+            tuple(row)
+            for row in (
+                spark.read.format("iceberg")
+                .load(table.location())
+                .select("id", "payload", "part")
+                .orderBy("part", "id", "payload")
+                .collect()
+            )
+        ]
+        assert rows == [(3, "keep-a", "A"), (2, "keep-b", "B")]
+
+        entries = _current_delete_entries(table_path)
+        assert len(entries) == 1
+        assert entries[0].data_file.partition == Record("A")
+    finally:
+        catalog.drop_table(identifier)
diff --git a/python/pysail/tests/spark/iceberg/test_iceberg_io.py b/python/pysail/tests/spark/iceberg/test_iceberg_io.py
index 5d012099c..07539a224 100644
--- a/python/pysail/tests/spark/iceberg/test_iceberg_io.py
+++ b/python/pysail/tests/spark/iceberg/test_iceberg_io.py
@@ -141,8 +141,60 @@ def test_iceberg_io_create_table_materializes_empty_metadata(spark, tmp_path):
         assert len(insert_metadata["snapshots"]) == 1
         assert len(insert_metadata["metadata-log"]) == 1
         assert insert_metadata["metadata-log"][0]["metadata-file"].endswith("/metadata/v1.metadata.json")
+        assert insert_metadata["metadata-log"][0]["timestamp-ms"] == create_metadata["last-updated-ms"]
         rows = spark.sql(f"SELECT id, name FROM {table_name} ORDER BY id").collect()  # noqa: S608
         assert [(row.id, row.name) for row in rows] == [(1, "one")]
+
+        spark.sql(f"INSERT INTO {table_name} VALUES (2, 'two')")  # noqa: S608
+        append_metadata = latest_iceberg_metadata(table_path)
+        assert append_metadata["metadata-log"][-1]["metadata-file"].endswith("/metadata/v2.metadata.json")
+        assert append_metadata["metadata-log"][-1]["timestamp-ms"] == insert_metadata["last-updated-ms"]
+
+        spark.sql(f"ALTER TABLE {table_name} SET TBLPROPERTIES ('custom.key' = 'value')")
+        altered_metadata = latest_iceberg_metadata(table_path)
+        assert altered_metadata["metadata-log"][-1]["metadata-file"].endswith("/metadata/v3.metadata.json")
+        assert altered_metadata["metadata-log"][-1]["timestamp-ms"] == append_metadata["last-updated-ms"]
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+
+
+def test_iceberg_merge_honors_external_write_data_path(spark, tmp_path):
+    table_path = tmp_path / "iceberg_external_write_path"
+    external_data_path = tmp_path / "external_data"
+    table_name = "iceberg_external_write_path_test"
+
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (id BIGINT)
+            USING ICEBERG
+            LOCATION '{escape_sql_string_literal(table_path.as_uri())}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read',
+              'write.data.path' = '{escape_sql_string_literal(external_data_path.as_uri())}'
+            )
+            """
+        )
+        spark.sql(f"INSERT INTO {table_name} VALUES (1), (2)")  # noqa: S608
+        spark.sql("CREATE OR REPLACE TEMP VIEW iceberg_external_write_source AS SELECT * FROM VALUES (1), (3) AS t(id)")
+        spark.sql(
+            """
+            MERGE INTO iceberg_external_write_path_test AS target
+            USING iceberg_external_write_source AS source
+            ON target.id = source.id
+            WHEN MATCHED THEN UPDATE SET id = source.id + 10
+            WHEN NOT MATCHED THEN INSERT (id) VALUES (source.id)
+            """
+        )
+
+        rows = spark.sql(f"SELECT id FROM {table_name} ORDER BY id").collect()  # noqa: S608
+        assert [row.id for row in rows] == [2, 3, 11]
+        assert not list(table_path.rglob("*.parquet"))
+        external_files = list(external_data_path.rglob("*.parquet"))
+        assert any(path.name.startswith("part-") for path in external_files)
+        assert any(path.name.startswith("delete-") for path in external_files)
     finally:
         spark.sql(f"DROP TABLE IF EXISTS {table_name}")
 
@@ -165,6 +217,8 @@ def test_iceberg_io_create_or_replace_existing_table_replaces_metadata_and_clear
             """
         )
         spark.sql(f"INSERT INTO {table_name} VALUES (1, 'one')")  # noqa: S608
+        insert_metadata = latest_iceberg_metadata(table_path)
+        previous_sequence_number = insert_metadata["last-sequence-number"]
 
         spark.sql(
             f"""
@@ -185,6 +239,54 @@ def test_iceberg_io_create_or_replace_existing_table_replaces_metadata_and_clear
         assert replacement_metadata["snapshots"] == []
         assert [field["name"] for field in replacement_metadata["schemas"][-1]["fields"]] == ["id", "name"]
         assert replacement_metadata["metadata-log"][-1]["metadata-file"].endswith("/metadata/v2.metadata.json")
+        assert replacement_metadata["metadata-log"][-1]["timestamp-ms"] == insert_metadata["last-updated-ms"]
+        assert replacement_metadata["last-sequence-number"] == previous_sequence_number
+
+        spark.sql(f"INSERT INTO {table_name} VALUES (2, 'two')")  # noqa: S608
+        post_replace_insert_metadata = latest_iceberg_metadata(table_path)
+        current_snapshot_id = post_replace_insert_metadata["current-snapshot-id"]
+        current_snapshot = next(
+            snapshot
+            for snapshot in post_replace_insert_metadata["snapshots"]
+            if snapshot["snapshot-id"] == current_snapshot_id
+        )
+        assert current_snapshot["sequence-number"] == previous_sequence_number + 1
+        assert post_replace_insert_metadata["last-sequence-number"] == previous_sequence_number + 1
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+
+
+def test_iceberg_io_create_or_replace_v3_table_preserves_next_row_id(spark, tmp_path):
+    table_path = tmp_path / "iceberg_replace_v3_row_id"
+    table_location = table_path.as_uri()
+    table_name = "iceberg_create_or_replace_v3_row_id_test"
+
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (id BIGINT)
+            USING ICEBERG
+            LOCATION '{escape_sql_string_literal(table_location)}'
+            TBLPROPERTIES ('format-version' = '3')
+            """
+        )
+        spark.sql(f"INSERT INTO {table_name} VALUES (1), (2), (3)")  # noqa: S608
+        insert_metadata = latest_iceberg_metadata(table_path)
+        assert insert_metadata["next-row-id"] == 3  # noqa: PLR2004
+
+        spark.sql(
+            f"""
+            CREATE OR REPLACE TABLE {table_name} (id BIGINT)
+            USING ICEBERG
+            LOCATION '{escape_sql_string_literal(table_location)}'
+            TBLPROPERTIES ('format-version' = '3')
+            """
+        )
+
+        replacement_metadata = latest_iceberg_metadata(table_path)
+        assert replacement_metadata["format-version"] == 3  # noqa: PLR2004
+        assert replacement_metadata["next-row-id"] == insert_metadata["next-row-id"]
     finally:
         spark.sql(f"DROP TABLE IF EXISTS {table_name}")
 
diff --git a/python/pysail/tests/spark/iceberg/test_iceberg_merge.py b/python/pysail/tests/spark/iceberg/test_iceberg_merge.py
new file mode 100644
index 000000000..a806e8e01
--- /dev/null
+++ b/python/pysail/tests/spark/iceberg/test_iceberg_merge.py
@@ -0,0 +1,884 @@
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import url2pathname
+
+import pyarrow.parquet as pq
+import pyspark.sql.functions as F  # noqa: N812
+import pytest
+from pyiceberg.io.pyarrow import PyArrowFileIO
+from pyiceberg.manifest import DataFileContent, ManifestContent, read_manifest_list
+from pyiceberg.typedef import Record
+from pyspark.sql.dataframe import DataFrame as SparkDataFrame
+
+from pysail.testing.spark.steps.iceberg import (
+    _current_manifest_list,
+    _current_snapshot,
+    _find_latest_metadata,
+    _latest_metadata_path,
+    _metadata_file_version,
+    _pyarrow_input_file,
+)
+from pysail.testing.spark.utils.sql import escape_sql_string_literal
+
+UNPARTITIONED_LAST_PARTITION_ID = 999
+
+
+def _uri_sql(path: Path) -> str:
+    return escape_sql_string_literal(path.as_uri())
+
+
+def _local_file_path(uri: str) -> Path:
+    parsed = urlparse(uri)
+    if parsed.scheme not in ("", "file"):
+        pytest.fail(f"cannot inspect non-local file URI: {uri}")
+    return Path(url2pathname(f"//{parsed.netloc}{parsed.path}" if parsed.netloc else parsed.path))
+
+
+def _drop_table(spark, name: str) -> None:
+    spark.sql(f"DROP TABLE IF EXISTS {name}")
+
+
+def _parquet_file_paths(table_path: Path) -> set[Path]:
+    return {path.relative_to(table_path) for path in table_path.rglob("*.parquet")}
+
+
+def _find_data_file_row_position(table_path: Path, target_id: int) -> tuple[Path, int]:
+    for relative_path in sorted(_parquet_file_paths(table_path)):
+        data_file = table_path / relative_path
+        row_offset = 0
+        for batch in pq.ParquetFile(data_file).iter_batches(columns=["id"]):
+            ids = batch.column("id").to_pylist()
+            if target_id in ids:
+                return data_file, row_offset + ids.index(target_id)
+            row_offset += batch.num_rows
+    pytest.fail(f"target id {target_id} was not found in Iceberg data files")
+
+
+def _current_manifests(table_path: Path) -> list[dict]:
+    metadata = _find_latest_metadata(table_path)
+    return _current_manifest_list(metadata)["manifests"]
+
+
+def _manifest_count(manifests: list[dict], *, content: str, key: str) -> int:
+    return sum(manifest.get(key) or 0 for manifest in manifests if manifest.get("content") == content)
+
+
+def _current_manifest_entries(table_path: Path, content: ManifestContent):
+    metadata = _find_latest_metadata(table_path)
+    snapshot = _current_snapshot(metadata)
+    io = PyArrowFileIO()
+    entries = []
+    for manifest in read_manifest_list(_pyarrow_input_file(io, snapshot["manifest-list"])):
+        if manifest.content == content:
+            entries.extend(manifest.fetch_manifest_entry(io))
+    return entries
+
+
+def _assert_current_snapshot_metadata(
+    table_path: Path,
+    metadata: dict,
+    *,
+    expected_metadata_version: int,
+    expected_snapshot_count: int,
+) -> dict:
+    snapshot = _current_snapshot(metadata)
+    assert metadata["current-snapshot-id"] == snapshot["snapshot-id"]
+    assert metadata["last-sequence-number"] == snapshot["sequence-number"]
+    assert metadata["last-partition-id"] == UNPARTITIONED_LAST_PARTITION_ID
+    assert metadata["refs"]["main"]["snapshot-id"] == snapshot["snapshot-id"]
+    assert metadata["snapshot-log"][-1]["snapshot-id"] == snapshot["snapshot-id"]
+    assert len(metadata["snapshots"]) == expected_snapshot_count
+    assert _metadata_file_version(_latest_metadata_path(table_path)) == expected_metadata_version
+    assert metadata["metadata-log"][-1]["metadata-file"].endswith(
+        f"/metadata/v{expected_metadata_version - 1}.metadata.json"
+    )
+    return snapshot
+
+
+def test_iceberg_merge_mor_update_delete_insert_writes_delete_manifest_and_reads_rows(spark, tmp_path):
+    table_name = "iceberg_merge_mor_rows"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING,
+              flag STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read',
+              'write.data.path' = 'custom_data'
+            )
+            """
+        )
+        spark.sql(
+            """
+            INSERT INTO iceberg_merge_mor_rows
+            SELECT * FROM VALUES
+              (1, 'old', 'keep'),
+              (2, 'old', 'update'),
+              (3, 'old', 'delete')
+            """
+        )
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_mor_source AS
+            SELECT * FROM VALUES
+              (2, 'new', 'insert'),
+              (3, 'ignored', 'delete'),
+              (4, 'ins', 'insert')
+            AS src(id, value, flag)
+            """
+        )
+
+        spark.sql(
+            """
+            MERGE INTO iceberg_merge_mor_rows AS t
+            USING iceberg_merge_mor_source AS s
+            ON t.id = s.id
+            WHEN MATCHED AND t.flag = 'update' THEN
+              UPDATE SET value = s.value
+            WHEN MATCHED AND t.flag = 'delete' THEN
+              DELETE
+            WHEN NOT MATCHED THEN
+              INSERT (id, value, flag) VALUES (s.id, s.value, s.flag)
+            """
+        )
+
+        rows = [
+            tuple(row) for row in spark.sql("SELECT id, value, flag FROM iceberg_merge_mor_rows ORDER BY id").collect()
+        ]
+        assert rows == [
+            (1, "old", "keep"),
+            (2, "new", "update"),
+            (4, "ins", "insert"),
+        ]
+
+        metadata = _find_latest_metadata(table_path)
+        current_snapshot = _assert_current_snapshot_metadata(
+            table_path,
+            metadata,
+            expected_metadata_version=3,
+            expected_snapshot_count=2,
+        )
+        assert current_snapshot["summary"]["operation"] == "overwrite"
+        assert current_snapshot["summary"]["total-records"] == "5"
+
+        manifests = _current_manifest_list(metadata)["manifests"]
+        expected_deleted_row_count = 2
+        data_manifests = [manifest for manifest in manifests if manifest.get("content") == "data"]
+        delete_manifests = [manifest for manifest in manifests if manifest.get("content") == "deletes"]
+        assert any(manifest["sequence-number"] < current_snapshot["sequence-number"] for manifest in data_manifests)
+        assert any(
+            manifest["sequence-number"] == current_snapshot["sequence-number"]
+            and (manifest.get("added-files-count") or 0) > 0
+            for manifest in data_manifests
+        )
+        assert all(manifest["sequence-number"] == current_snapshot["sequence-number"] for manifest in delete_manifests)
+        assert _manifest_count(manifests, content="deletes", key="added-files-count") >= 1
+        assert _manifest_count(manifests, content="deletes", key="added-rows-count") >= expected_deleted_row_count
+        assert _manifest_count(manifests, content="data", key="deleted-files-count") == 0
+        assert _manifest_count(manifests, content="data", key="deleted-rows-count") == 0
+
+        custom_prefix = f"{(table_path / 'custom_data').as_uri()}/"
+        data_entries = _current_manifest_entries(table_path, ManifestContent.DATA)
+        delete_entries = _current_manifest_entries(table_path, ManifestContent.DELETES)
+        assert data_entries
+        assert delete_entries
+        assert current_snapshot["summary"]["total-data-files"] == str(len(data_entries))
+        assert current_snapshot["summary"]["total-delete-files"] == str(len(delete_entries))
+        assert all(entry.data_file.file_path.startswith(custom_prefix) for entry in data_entries)
+        assert all(entry.data_file.file_path.startswith(f"{custom_prefix}delete-") for entry in delete_entries)
+        referenced_data_files = {entry.data_file.file_path for entry in data_entries}
+        deleted_row_file_paths = [
+            path
+            for entry in delete_entries
+            for path in pq.read_table(_local_file_path(entry.data_file.file_path), columns=["file_path"])
+            .column("file_path")
+            .to_pylist()
+        ]
+        assert deleted_row_file_paths
+        assert all(entry.data_file.content == DataFileContent.POSITION_DELETES for entry in delete_entries)
+        assert all(path in referenced_data_files for path in deleted_row_file_paths)
+        assert all(entry.data_file.sort_order_id is None for entry in delete_entries)
+        assert all(not entry.data_file.equality_ids for entry in delete_entries)
+        assert all(getattr(entry.data_file, "content_offset", None) is None for entry in delete_entries)
+        assert all(getattr(entry.data_file, "content_size_in_bytes", None) is None for entry in delete_entries)
+        assert sum(entry.data_file.record_count for entry in delete_entries) >= expected_deleted_row_count
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_dataframe_merge_into_mor_update_delete_insert(spark, tmp_path):
+    if not hasattr(SparkDataFrame, "mergeInto"):
+        pytest.skip("DataFrame.mergeInto requires Spark 4.0+ (missing in this PySpark)")
+
+    table_name = "iceberg_dataframe_merge_mor"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING,
+              flag STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql(
+            """
+            INSERT INTO iceberg_dataframe_merge_mor
+            SELECT * FROM VALUES
+              (1, 'old', 'keep'),
+              (2, 'old', 'update'),
+              (3, 'old', 'delete')
+            """
+        )
+
+        source_df = spark.createDataFrame(
+            [(2, "new", "insert"), (3, "ignored", "delete"), (4, "ins", "insert")],
+            "src_id INT, src_value STRING, src_flag STRING",
+        )
+        (
+            source_df.mergeInto(table_name, F.expr("id = src_id"))
+            .whenMatched(F.expr("flag = 'update'"))
+            .update(assignments={"value": F.col("src_value")})
+            .whenMatched(F.expr("flag = 'delete'"))
+            .delete()
+            .whenNotMatched()
+            .insert(
+                assignments={
+                    "id": F.col("src_id"),
+                    "value": F.col("src_value"),
+                    "flag": F.col("src_flag"),
+                }
+            )
+            .merge()
+        )
+
+        rows = [
+            tuple(row)
+            for row in spark.sql("SELECT id, value, flag FROM iceberg_dataframe_merge_mor ORDER BY id").collect()
+        ]
+        assert rows == [
+            (1, "old", "keep"),
+            (2, "new", "update"),
+            (4, "ins", "insert"),
+        ]
+
+        metadata = _find_latest_metadata(table_path)
+        snapshot = _assert_current_snapshot_metadata(
+            table_path,
+            metadata,
+            expected_metadata_version=3,
+            expected_snapshot_count=2,
+        )
+        assert snapshot["summary"]["operation"] == "overwrite"
+        assert snapshot["summary"]["added-position-deletes"] == "2"
+        assert snapshot["summary"]["total-records"] == "5"
+        data_entries = _current_manifest_entries(table_path, ManifestContent.DATA)
+        delete_entries = _current_manifest_entries(table_path, ManifestContent.DELETES)
+        assert snapshot["summary"]["total-data-files"] == str(len(data_entries))
+        assert snapshot["summary"]["total-delete-files"] == str(len(delete_entries))
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_rejects_multiple_source_rows_for_one_target_row(spark, tmp_path):
+    table_name = "iceberg_merge_cardinality"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_cardinality VALUES (1, 'target')")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_cardinality_source AS
+            SELECT * FROM VALUES
+              (1, 'source-a'),
+              (1, 'source-b')
+            AS src(id, value)
+            """
+        )
+
+        with pytest.raises(Exception, match=r"MERGE_CARDINALITY_VIOLATION|Multiple source rows"):
+            spark.sql(
+                f"""
+                MERGE INTO {table_name} AS t
+                USING iceberg_merge_cardinality_source AS s
+                ON t.id = s.id
+                WHEN MATCHED THEN
+                  UPDATE SET value = s.value
+                WHEN NOT MATCHED THEN
+                  INSERT *
+                """
+            ).collect()
+
+        rows = [tuple(row) for row in spark.sql("SELECT id, value FROM iceberg_merge_cardinality").collect()]
+        assert rows == [(1, "target")]
+        assert len(_current_manifests(table_path)) == 1
+    finally:
+        _drop_table(spark, table_name)
+
+
+@pytest.mark.parametrize("merge_mode", [None, "copy-on-write"], ids=["default", "explicit-cow"])
+def test_iceberg_merge_rejects_copy_on_write_before_writing_empty_table(spark, tmp_path, merge_mode):
+    table_name = "iceberg_merge_copy_on_write_reject"
+    table_path = tmp_path / table_name
+    mode_property = "" if merge_mode is None else f", 'write.merge.mode' = '{merge_mode}'"
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES ('format-version' = '2'{mode_property})
+            """
+        )
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_copy_on_write_source AS
+            SELECT * FROM VALUES (1, 'new') AS src(id, value)
+            """
+        )
+        before_metadata_path = _latest_metadata_path(table_path)
+        before_parquet_files = _parquet_file_paths(table_path)
+
+        with pytest.raises(Exception, match=r"write\.merge\.mode=copy-on-write|copy-on-write.*not supported"):
+            spark.sql(
+                f"""
+                MERGE INTO {table_name} AS t
+                USING iceberg_merge_copy_on_write_source AS s
+                ON t.id = s.id
+                WHEN NOT MATCHED THEN INSERT *
+                """
+            ).collect()
+
+        assert spark.sql("SELECT * FROM iceberg_merge_copy_on_write_reject").collect() == []
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        assert _parquet_file_paths(table_path) == before_parquet_files
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_rejects_position_deletes_on_v1_table(spark, tmp_path):
+    table_name = "iceberg_merge_v1_position_delete"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '1',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_v1_position_delete VALUES (1, 'old')")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_v1_source AS
+            SELECT * FROM VALUES (1, 'new') AS src(id, value)
+            """
+        )
+        before_metadata_path = _latest_metadata_path(table_path)
+        before_parquet_files = _parquet_file_paths(table_path)
+
+        with pytest.raises(Exception, match=r"format-version 2|position delete writes"):
+            spark.sql(
+                f"""
+                MERGE INTO {table_name} AS t
+                USING iceberg_merge_v1_source AS s
+                ON t.id = s.id
+                WHEN MATCHED THEN
+                  UPDATE SET value = s.value
+                """
+            ).collect()
+
+        rows = [tuple(row) for row in spark.sql("SELECT id, value FROM iceberg_merge_v1_position_delete").collect()]
+        assert rows == [(1, "old")]
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        assert _parquet_file_paths(table_path) == before_parquet_files
+        assert len(_find_latest_metadata(table_path)["snapshots"]) == 1
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_rejects_position_deletes_on_v3_table_without_metadata_commit(spark, tmp_path):
+    table_name = "iceberg_merge_v3_position_delete"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '3',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_v3_position_delete VALUES (1, 'old')")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_v3_source AS
+            SELECT * FROM VALUES (1, 'new') AS src(id, value)
+            """
+        )
+        before_metadata_path = _latest_metadata_path(table_path)
+        before_parquet_files = _parquet_file_paths(table_path)
+
+        with pytest.raises(Exception, match=r"deletion vectors|v3 MERGE MOR position delete"):
+            spark.sql(
+                f"""
+                MERGE INTO {table_name} AS t
+                USING iceberg_merge_v3_source AS s
+                ON t.id = s.id
+                WHEN MATCHED THEN
+                  UPDATE SET value = s.value
+                """
+            ).collect()
+
+        rows = [tuple(row) for row in spark.sql("SELECT id, value FROM iceberg_merge_v3_position_delete").collect()]
+        assert rows == [(1, "old")]
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        assert _parquet_file_paths(table_path) == before_parquet_files
+        assert len(_find_latest_metadata(table_path)["snapshots"]) == 1
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_with_schema_evolution_is_rejected_without_side_effects(spark, tmp_path):
+    table_name = "iceberg_merge_schema_evolution_reject"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_schema_evolution_reject VALUES (1, 'old')")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_schema_evolution_source AS
+            SELECT 1 AS id, 'new' AS value, 'extra' AS extra
+            """
+        )
+        before_metadata_path = _latest_metadata_path(table_path)
+        before_parquet_files = _parquet_file_paths(table_path)
+
+        with pytest.raises(Exception, match=r"Iceberg MERGE WITH SCHEMA EVOLUTION.*not supported"):
+            spark.sql(
+                f"""
+                MERGE WITH SCHEMA EVOLUTION INTO {table_name} AS t
+                USING iceberg_merge_schema_evolution_source AS s
+                ON t.id = s.id
+                WHEN MATCHED THEN UPDATE SET *
+                WHEN NOT MATCHED THEN INSERT *
+                """
+            ).collect()
+
+        assert spark.table(table_name).schema.names == ["id", "value"]
+        rows = [
+            tuple(row) for row in spark.sql("SELECT id, value FROM iceberg_merge_schema_evolution_reject").collect()
+        ]
+        assert rows == [(1, "old")]
+        assert _latest_metadata_path(table_path) == before_metadata_path
+        assert _parquet_file_paths(table_path) == before_parquet_files
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_updates_rows_beyond_the_first_input_batch(spark, tmp_path):
+    table_name = "iceberg_merge_multi_batch"
+    table_path = tmp_path / table_name
+    target_id = 49_999
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id BIGINT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_multi_batch SELECT id, 'old' FROM range(50000)")
+        _, target_position = _find_data_file_row_position(table_path, target_id)
+        assert target_position >= 8192  # noqa: PLR2004
+        spark.sql(
+            f"""
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_multi_batch_source AS
+            SELECT {target_id}L AS id, 'updated' AS value
+            """
+        )
+
+        spark.sql(
+            f"""
+            MERGE INTO {table_name} AS t
+            USING iceberg_merge_multi_batch_source AS s
+            ON t.id = s.id
+            WHEN MATCHED THEN UPDATE SET value = s.value
+            """
+        ).collect()
+
+        rows = [
+            tuple(row)
+            for row in spark.sql(
+                f"SELECT id, value FROM iceberg_merge_multi_batch WHERE id = {target_id}"  # noqa: S608
+            ).collect()
+        ]
+        assert rows == [(target_id, "updated")]
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_uses_absolute_positions_for_large_multi_row_group_file(spark, tmp_path):
+    table_name = "iceberg_merge_split_file_position"
+    table_path = tmp_path / table_name
+    target_id = 4_300_000
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id BIGINT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        large_file_insert_sql = (
+            f"INSERT INTO {table_name} "  # noqa: S608
+            "SELECT id, sha2(CAST(id AS STRING), 256) AS value "
+            "FROM range(4400000)"
+        )
+        spark.sql(large_file_insert_sql)
+
+        data_file, expected_position = _find_data_file_row_position(table_path, target_id)
+        parquet_file = pq.ParquetFile(data_file)
+        assert data_file.stat().st_size > 10 * 1024 * 1024
+        assert parquet_file.metadata.num_row_groups >= 2  # noqa: PLR2004
+        assert expected_position >= parquet_file.metadata.row_group(0).num_rows
+
+        spark.sql(
+            f"""
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_split_file_source AS
+            SELECT {target_id}L AS id, 'updated' AS value
+            """
+        )
+        spark.sql(
+            f"""
+            MERGE INTO {table_name} AS t
+            USING iceberg_merge_split_file_source AS s
+            ON t.id = s.id
+            WHEN MATCHED THEN UPDATE SET value = s.value
+            """
+        ).collect()
+
+        target_row_sql = f"SELECT id, value FROM {table_name} WHERE id = {target_id}"  # noqa: S608
+        rows = [tuple(row) for row in spark.sql(target_row_sql).collect()]
+        assert rows == [(target_id, "updated")]
+
+        delete_entries = _current_manifest_entries(table_path, ManifestContent.DELETES)
+        positions = sorted(
+            position
+            for entry in delete_entries
+            for position in pq.read_table(_local_file_path(entry.data_file.file_path), columns=["pos"])
+            .column("pos")
+            .to_pylist()
+        )
+        assert positions == [expected_position]
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_metadata_as_data_read_preserves_row_level_metadata(spark, tmp_path):
+    table_name = "iceberg_merge_metadata_as_data"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            OPTIONS (metadataAsDataRead 'true')
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_metadata_as_data VALUES (1, 'old')")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_metadata_as_data_source AS
+            SELECT 1 AS id, 'updated' AS value
+            """
+        )
+
+        spark.sql(
+            f"""
+            MERGE INTO {table_name} AS t
+            USING iceberg_merge_metadata_as_data_source AS s
+            ON t.id = s.id
+            WHEN MATCHED THEN UPDATE SET value = s.value
+            """
+        ).collect()
+
+        rows = [
+            tuple(row) for row in spark.read.format("iceberg").load(table_path.as_uri()).select("id", "value").collect()
+        ]
+        assert rows == [(1, "updated")]
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_position_delete_round_trips_null_partition_value(spark, tmp_path):
+    table_name = "iceberg_merge_null_partition"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING,
+              part STRING
+            )
+            USING iceberg
+            PARTITIONED BY (part)
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_null_partition VALUES (1, 'old', NULL)")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_null_partition_source AS
+            SELECT 1 AS id, 'updated' AS value
+            """
+        )
+
+        spark.sql(
+            f"""
+            MERGE INTO {table_name} AS t
+            USING iceberg_merge_null_partition_source AS s
+            ON t.id = s.id
+            WHEN MATCHED THEN UPDATE SET value = s.value
+            """
+        ).collect()
+
+        rows = [tuple(row) for row in spark.sql("SELECT id, value, part FROM iceberg_merge_null_partition").collect()]
+        assert rows == [(1, "updated", None)]
+        delete_entries = _current_manifest_entries(table_path, ManifestContent.DELETES)
+        assert len(delete_entries) == 1
+        assert delete_entries[0].data_file.partition == Record(None)
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_merge_row_delta_snapshot_operation_tracks_written_files(spark, tmp_path):
+    table_name = "iceberg_merge_row_delta_operation"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql(
+            """
+            INSERT INTO iceberg_merge_row_delta_operation VALUES
+              (1, 'keep'),
+              (2, 'delete')
+            """
+        )
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_row_delta_source AS
+            SELECT 3 AS id, 'insert' AS value
+            """
+        )
+
+        spark.sql(
+            """
+            MERGE INTO iceberg_merge_row_delta_operation AS t
+            USING iceberg_merge_row_delta_source AS s
+            ON t.id = s.id
+            WHEN NOT MATCHED THEN INSERT (id, value) VALUES (s.id, s.value)
+            """
+        ).collect()
+
+        insert_snapshot = _current_snapshot(_find_latest_metadata(table_path))
+        assert insert_snapshot["summary"]["operation"] == "append"
+
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_row_delta_source AS
+            SELECT 2 AS id, 'ignored' AS value
+            """
+        )
+        spark.sql(
+            """
+            MERGE INTO iceberg_merge_row_delta_operation AS t
+            USING iceberg_merge_row_delta_source AS s
+            ON t.id = s.id
+            WHEN MATCHED THEN DELETE
+            """
+        ).collect()
+
+        delete_snapshot = _current_snapshot(_find_latest_metadata(table_path))
+        assert delete_snapshot["summary"]["operation"] == "delete"
+        rows = [
+            tuple(row)
+            for row in spark.sql("SELECT id, value FROM iceberg_merge_row_delta_operation ORDER BY id").collect()
+        ]
+        assert rows == [(1, "keep"), (3, "insert")]
+    finally:
+        _drop_table(spark, table_name)
+
+
+def test_iceberg_noop_merge_reuses_parent_manifests_with_overwrite_snapshot(spark, tmp_path):
+    table_name = "iceberg_merge_noop_snapshot"
+    table_path = tmp_path / table_name
+
+    _drop_table(spark, table_name)
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              value STRING
+            )
+            USING iceberg
+            LOCATION '{_uri_sql(table_path)}'
+            TBLPROPERTIES (
+              'format-version' = '2',
+              'write.merge.mode' = 'merge-on-read'
+            )
+            """
+        )
+        spark.sql("INSERT INTO iceberg_merge_noop_snapshot VALUES (1, 'keep')")
+        spark.sql(
+            """
+            CREATE OR REPLACE TEMP VIEW iceberg_merge_noop_source AS
+            SELECT 2 AS id, 'unmatched' AS value
+            """
+        )
+
+        before_metadata = _find_latest_metadata(table_path)
+        before_manifest_paths = {manifest["manifest-path"] for manifest in _current_manifests(table_path)}
+        before_manifest_files = set((table_path / "metadata").glob("manifest-*.avro"))
+
+        spark.sql(
+            f"""
+            MERGE INTO {table_name} AS t
+            USING iceberg_merge_noop_source AS s
+            ON t.id = s.id
+            WHEN MATCHED THEN UPDATE SET value = s.value
+            """
+        ).collect()
+
+        after_metadata = _find_latest_metadata(table_path)
+        after_snapshot = _current_snapshot(after_metadata)
+        after_manifest_paths = {manifest["manifest-path"] for manifest in _current_manifests(table_path)}
+        after_manifest_files = set((table_path / "metadata").glob("manifest-*.avro"))
+        assert len(after_metadata["snapshots"]) == len(before_metadata["snapshots"]) + 1
+        assert after_snapshot["summary"]["operation"] == "overwrite"
+        assert after_manifest_paths == before_manifest_paths
+        assert after_manifest_files == before_manifest_files
+        rows = [tuple(row) for row in spark.sql("SELECT id, value FROM iceberg_merge_noop_snapshot").collect()]
+        assert rows == [(1, "keep")]
+    finally:
+        _drop_table(spark, table_name)
diff --git a/python/pysail/tests/spark/iceberg/test_iceberg_partitioned_writes.py b/python/pysail/tests/spark/iceberg/test_iceberg_partitioned_writes.py
index 3f953aa43..545d91e71 100644
--- a/python/pysail/tests/spark/iceberg/test_iceberg_partitioned_writes.py
+++ b/python/pysail/tests/spark/iceberg/test_iceberg_partitioned_writes.py
@@ -248,6 +248,43 @@ def test_iceberg_partition_writes_sql(spark, tmp_path):
         spark.sql("DROP TABLE IF EXISTS tmp_ice_multi")
 
 
+@pytest.mark.skipif(platform.system() == "Windows", reason="may not work on Windows")
+def test_identity_partition_distinguishes_null_from_null_string(spark, tmp_path):
+    table_name = "iceberg_partition_null_identity"
+    table_path = tmp_path / table_name
+    spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+    try:
+        spark.sql(
+            f"""
+            CREATE TABLE {table_name} (
+              id INT,
+              part STRING
+            )
+            USING iceberg
+            PARTITIONED BY (part)
+            LOCATION '{escape_sql_string_literal(table_path.as_uri())}'
+            """
+        )
+        partition_rows_insert_sql = (
+            f"INSERT INTO {table_name} VALUES "  # noqa: S608
+            "(1, NULL), "
+            "(2, 'null')"
+        )
+        spark.sql(partition_rows_insert_sql)
+
+        null_partition_rows_sql = f"SELECT id FROM {table_name} WHERE part IS NULL"  # noqa: S608
+        literal_null_partition_rows_sql = f"SELECT id FROM {table_name} WHERE part = 'null'"  # noqa: S608
+        null_rows = [row.id for row in spark.sql(null_partition_rows_sql).collect()]
+        string_rows = [row.id for row in spark.sql(literal_null_partition_rows_sql).collect()]
+        assert null_rows == [1]
+        assert string_rows == [2]
+
+        partition_files = list((table_path / "data" / "part=null").glob("*.parquet"))
+        assert len(partition_files) == 2  # noqa: PLR2004
+    finally:
+        spark.sql(f"DROP TABLE IF EXISTS {table_name}")
+
+
 @pytest.mark.skipif(platform.system() == "Windows", reason="may not work on Windows")
 def test_partitioned_append_infers_spec_from_metadata(spark, tmp_path):
     catalog = create_sql_catalog(tmp_path)
diff --git a/python/pysail/tests/spark/iceberg/test_iceberg_write.py b/python/pysail/tests/spark/iceberg/test_iceberg_write.py
index 7b852fbf2..59c166a9b 100644
--- a/python/pysail/tests/spark/iceberg/test_iceberg_write.py
+++ b/python/pysail/tests/spark/iceberg/test_iceberg_write.py
@@ -137,6 +137,37 @@ def test_iceberg_write_append_mode(spark, sql_catalog):
         sql_catalog.drop_table(identifier)
 
 
+def test_iceberg_write_honors_absolute_data_path(spark, sql_catalog, tmp_path):
+    identifier = "default.write_absolute_data_path"
+    table_path = tmp_path / "absolute_data_path_table"
+    data_path = tmp_path / "absolute_data_path_files"
+    table = sql_catalog.create_table(
+        identifier=identifier,
+        location=table_path.as_uri(),
+        schema=Schema(
+            NestedField(field_id=1, name="id", field_type=LongType(), required=False),
+            NestedField(field_id=2, name="value", field_type=StringType(), required=False),
+        ),
+        properties={"write.data.path": data_path.as_uri()},
+    )
+    try:
+        df = spark.createDataFrame([(1, "v0")], schema="id LONG, value STRING")
+        df.write.format("iceberg").mode("append").save(table.location())
+
+        data_files = set(data_path.rglob("*.parquet"))
+        assert data_files
+        assert not set(table_path.rglob("*.parquet"))
+
+        static_table = StaticTable.from_metadata(table.location(), properties=pyiceberg_file_io_properties())
+        manifest_data_files = {task.file.file_path for task in static_table.scan().plan_files()}
+        assert manifest_data_files == {path.absolute().as_uri() for path in data_files}
+
+        rows = [tuple(row) for row in spark.read.format("iceberg").load(table.location()).collect()]
+        assert rows == [(1, "v0")]
+    finally:
+        sql_catalog.drop_table(identifier)
+
+
 def test_iceberg_sql_read_after_write(spark, sql_catalog):
     identifier = "default.write_sql_table"
     table = sql_catalog.create_table(
diff --git a/python/pysail/tests/spark/optimizer/test_features.py b/python/pysail/tests/spark/optimizer/test_features.py
index 6eb05fa63..8ae9c40d3 100644
--- a/python/pysail/tests/spark/optimizer/test_features.py
+++ b/python/pysail/tests/spark/optimizer/test_features.py
@@ -9,7 +9,7 @@
 pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail only")
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="package")
 def remote():
     with spark_connect_server(envs={"SAIL_OPTIMIZER__ENABLE_JOIN_REORDER": "true"}) as server:
         yield server.remote
diff --git a/python/pysail/tests/spark/session/__init__.py b/python/pysail/tests/spark/session/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/python/pysail/tests/spark/session/test_heartbeat.py b/python/pysail/tests/spark/session/test_heartbeat.py
new file mode 100644
index 000000000..24c5bbdc0
--- /dev/null
+++ b/python/pysail/tests/spark/session/test_heartbeat.py
@@ -0,0 +1,74 @@
+import time
+
+import pyspark.sql.functions as F  # noqa: N812
+import pytest
+from pyspark.errors.exceptions.connect import SparkConnectGrpcException
+from pyspark.sql.types import LongType, Row
+
+from pysail.testing.spark.session import spark_connect_server, spark_session_factory
+
+pytestmark = pytest.mark.skip(reason="slow tests that should be run manually")
+
+
+@pytest.fixture(scope="module")
+def bad_sessions():
+    envs = {
+        "SAIL_SPARK__SESSION_TIMEOUT_SECS": "2",
+        "SAIL_SPARK__EXECUTION_HEARTBEAT_INTERVAL_SECS": "4",
+    }
+    with spark_connect_server(envs=envs) as server, spark_session_factory(server.remote) as sessions:
+        yield sessions
+
+
+@pytest.fixture(scope="module")
+def good_sessions():
+    envs = {
+        "SAIL_SPARK__SESSION_TIMEOUT_SECS": "2",
+        "SAIL_SPARK__EXECUTION_HEARTBEAT_INTERVAL_SECS": "1",
+    }
+    with spark_connect_server(envs=envs) as server, spark_session_factory(server.remote) as sessions:
+        yield sessions
+
+
+@pytest.fixture(scope="module")
+def identity():
+    @F.udf(LongType())
+    def _identity(value):
+        time.sleep(5)
+        return value
+
+    return _identity
+
+
+@pytest.fixture(scope="module")
+def data(identity):
+    def _data(spark):
+        return spark.range(1).select(identity("id").alias("id"))
+
+    return _data
+
+
+def test_query_fails_when_session_timeout_precedes_heartbeat(bad_sessions, data):
+    spark = bad_sessions.create()
+    with pytest.raises(SparkConnectGrpcException):
+        data(spark).collect()
+
+
+def test_command_fails_when_session_timeout_precedes_heartbeat(bad_sessions, data):
+    spark = bad_sessions.create()
+    with pytest.raises(SparkConnectGrpcException):
+        data(spark).write.format("noop").mode("overwrite").save()
+
+
+def test_query_succeeds_when_heartbeat_precedes_session_timeout(good_sessions, data):
+    spark = good_sessions.create()
+    assert data(spark).collect() == [Row(id=0)]
+    # Send another request to make sure the session is still alive.
+    assert spark.range(1).count() == 1
+
+
+def test_command_succeeds_when_heartbeat_precedes_session_timeout(good_sessions, data):
+    spark = good_sessions.create()
+    data(spark).write.format("noop").mode("overwrite").save()
+    # Send another request to make sure the session is still alive.
+    assert spark.range(1).count() == 1
diff --git a/python/pysail/tests/spark/session/test_reattachable_execution.py b/python/pysail/tests/spark/session/test_reattachable_execution.py
new file mode 100644
index 000000000..8d6a6ae8c
--- /dev/null
+++ b/python/pysail/tests/spark/session/test_reattachable_execution.py
@@ -0,0 +1,31 @@
+import pyspark.sql.connect.functions as F  # noqa: N812
+import pytest
+from pyspark.errors.exceptions.connect import SparkConnectGrpcException
+
+from pysail.testing.spark.session import spark_session_factory
+from pysail.testing.spark.utils.common import is_jvm_spark
+
+pytestmark = pytest.mark.skipif(is_jvm_spark(), reason="Sail only")
+
+
+@pytest.fixture(scope="module")
+def fail():
+    @F.udf("long")
+    def _fail(_value):
+        msg = "expected failure"
+        raise RuntimeError(msg)
+
+    return _fail
+
+
+def test_execution_error_without_reattachment(remote, fail):
+    """Error handling for reattachable execution (the default) is exercised frequently in other tests.
+    This test ensures that the error can still be propagated for (non-default) non-reattachable execution.
+    """
+    with spark_session_factory(remote) as sessions:
+        spark = sessions.create()
+        spark._client.disable_reattachable_execute()  # noqa: SLF001
+        with pytest.raises(SparkConnectGrpcException, match="expected failure"):
+            spark.range(1).select(fail("id")).write.format("noop").mode("overwrite").save()
+        with pytest.raises(SparkConnectGrpcException, match="expected failure"):
+            spark.range(1).select(fail("id")).collect()
diff --git a/python/pysail/tests/spark/test_basic.py b/python/pysail/tests/spark/test_basic.py
index bdf9bbe0b..6cb26e7d7 100644
--- a/python/pysail/tests/spark/test_basic.py
+++ b/python/pysail/tests/spark/test_basic.py
@@ -359,6 +359,30 @@ def test_explode(spark):
     )
 
 
+def test_inline_null_struct(spark):
+    data = spark.createDataFrame(
+        [
+            Row(structlist=[Row(b=1, c=2), Row(b=3, c=4)]),
+            Row(structlist=[Row(b=None, c=5), None]),
+            Row(structlist=[]),
+        ]
+    )
+
+    assert [tuple(x) for x in data.select(F.inline("structlist")).collect()] == [
+        (1, 2),
+        (3, 4),
+        (None, 5),
+        (None, None),
+    ]
+    assert [tuple(x) for x in data.select(F.inline_outer("structlist")).collect()] == [
+        (1, 2),
+        (3, 4),
+        (None, 5),
+        (None, None),
+        (None, None),
+    ]
+
+
 def test_udf(df, udf_add_one, udf_add_x_y, udf_add):
     assert_frame_equal(
         df.sort("a").limit(1).select(udf_add_one(F.col("a"))).toPandas(),
@@ -382,6 +406,15 @@ def test_udf(df, udf_add_one, udf_add_x_y, udf_add):
     )
 
 
+def test_udf_binary_view_input(spark, tmp_path):
+    path = str(tmp_path / "binary.parquet")
+    spark.createDataFrame([(b"value",)], "value BINARY").write.parquet(path)
+
+    identity = F.udf(lambda value: value, "binary")
+    actual = spark.read.parquet(path).select(identity("value").alias("value")).collect()
+    assert actual == [Row(value=b"value")]
+
+
 def test_sql_with_clause(spark, df, df_view):
     assert_frame_equal(
         spark.sql(f"WITH test AS (SELECT * FROM {df_view}) SELECT * FROM test").toPandas(),  # noqa: S608
diff --git a/python/pysail/tests/spark/test_clickbench.py b/python/pysail/tests/spark/test_clickbench.py
index b0e27dd57..00ba80b61 100644
--- a/python/pysail/tests/spark/test_clickbench.py
+++ b/python/pysail/tests/spark/test_clickbench.py
@@ -134,7 +134,7 @@ def data(spark, tmp_path_factory):
     pq.write_table(table, data_path)
     df = spark.read.parquet(data_path)
     df = df.withColumn("EventTime", F.col("EventTime").cast("timestamp"))
-    df = df.withColumn("EventDate", F.date_add(F.lit("1970-01-01"), F.col("EventDate")))
+    df = df.withColumn("EventDate", F.col("EventDate").cast("int").cast("date"))
     df.createOrReplaceTempView("hits")
     yield
     spark.catalog.dropTempView("hits")
diff --git a/python/pysail/tests/spark/test_config.py b/python/pysail/tests/spark/test_config.py
index 0de285ff9..87db5630a 100644
--- a/python/pysail/tests/spark/test_config.py
+++ b/python/pysail/tests/spark/test_config.py
@@ -5,8 +5,15 @@
 from pysail.spark import SparkConnectServer
 
 
+@pytest.fixture(scope="module", autouse=True)
+def server():
+    # The tests below need the global state to be initialized while the environment variables
+    # are still pristine. This fixture ensures this even when running the tests against a server from `SPARK_REMOTE`.
+    _ = SparkConnectServer()
+
+
 def test_warning_on_static_config_addition(monkeypatch):
-    key = "SAIL_TELEMETRY__OTLP_ENDPOINT"
+    key = "SAIL_TELEMETRY__EXPORTER__OTLP__ENDPOINT"
     assert key not in os.environ
     monkeypatch.setenv(key, "http://localhost:4317")
     with pytest.warns(RuntimeWarning, match=f"ignored.*{key}"):
diff --git a/python/pysail/tests/spark/udf/test_udf_kwargs.py b/python/pysail/tests/spark/udf/test_udf_kwargs.py
index 4cc0ebba2..055ac6fed 100644
--- a/python/pysail/tests/spark/udf/test_udf_kwargs.py
+++ b/python/pysail/tests/spark/udf/test_udf_kwargs.py
@@ -109,14 +109,23 @@ def test_udtf_kwargs_reversed_order(udtf_concat):
 
 
 def test_udtf_row_output_invalid_scalar_raises_pickle_exception():
-    from pyspark.errors.exceptions.connect import PickleException
+    if pyspark_version() >= (4, 2):
+        from pyspark.errors.exceptions.connect import PythonException
+
+        exception = PythonException
+        match = "UDTF_ARROW_DATA_CONVERSION_ERROR"
+    else:
+        from pyspark.errors.exceptions.connect import PickleException
+
+        exception = PickleException
+        match = "PickleException"
 
     @udtf(returnType="x: boolean")
     class RowOutputUDTF:
         def eval(self):
             yield (Row(a=0, b=1.1, c=2),)
 
-    with pytest.raises(PickleException, match="PickleException"):
+    with pytest.raises(exception, match=match):
         RowOutputUDTF().collect()
 
 
@@ -132,10 +141,30 @@ class ArrowOutputUDTF:
         def eval(self):
             yield (1,)
 
-    with pytest.raises(PythonException, match="UDTF_ARROW_TYPE_CONVERSION_ERROR"):
+    with pytest.raises(PythonException, match=r"UDTF_ARROW_(TYPE|DATA)_CONVERSION_ERROR"):
         ArrowOutputUDTF().collect()
 
 
+@pytest.mark.parametrize("use_arrow", [False, True])
+def test_arrow_udtf_lateral_view_uses_argument_schema(spark, use_arrow):
+    """Passthrough columns must not shift Arrow UDTF argument conversions."""
+
+    @udtf(returnType="output: string", useArrow=use_arrow)
+    class EchoUDTF:
+        def eval(self, value):
+            yield (value,)
+
+    function_name = f"echo_udtf_{'' if use_arrow else 'not_'}use_arrow"
+    spark.udtf.register(function_name, EchoUDTF)
+
+    df = spark.sql(
+        f"""SELECT passthrough, value, output
+        FROM VALUES (X'706173737468726f756768', 'value') AS input(passthrough, value)
+        LATERAL VIEW {function_name}(value) AS output"""  # noqa: S608
+    )
+    assert df.collect() == [Row(passthrough=b"passthrough", value="value", output="value")]
+
+
 @pytest.mark.skipif(
     pyspark_version() < (4, 1),
     reason="UDTF analyze tests require PySpark 4.1+",
diff --git a/python/pysail/tests/spark/udf/test_view_type_inputs.py b/python/pysail/tests/spark/udf/test_view_type_inputs.py
new file mode 100644
index 000000000..b3a102dca
--- /dev/null
+++ b/python/pysail/tests/spark/udf/test_view_type_inputs.py
@@ -0,0 +1,184 @@
+from collections.abc import Iterator
+
+import pandas as pd
+import pyarrow as pa
+import pyspark.sql.functions as F  # noqa: N812
+import pytest
+from pyspark.sql import Row, Window
+from pyspark.sql.functions import PandasUDFType, pandas_udf, udtf
+from pyspark.sql.types import BinaryType, StringType, StructType
+
+from pysail.testing.spark.utils.common import pyspark_version
+
+VALUES = [(0, b"", ""), (1, b"x" * 32, "y" * 32), (2, None, None)]
+
+
+@pytest.fixture(scope="module")
+def view_type_df(spark, tmp_path_factory):
+    path = str(tmp_path_factory.mktemp("view-types") / "input.parquet")
+    spark.createDataFrame(VALUES, "id int, value binary, text string").write.parquet(path)
+    return spark.read.parquet(path)
+
+
+def nested_value():
+    return F.struct(
+        "value",
+        "text",
+        F.array("value").alias("values"),
+        F.create_map("text", "value").alias("mapping"),
+    )
+
+
+def expected_rows():
+    return [Row(id=id_, value=value, text=text) for id_, value, text in VALUES]
+
+
+def run_table_udtf(spark, df, name, function):
+    view_name = f"{name}_input"
+    spark.udtf.register(name, function)
+    df.createOrReplaceTempView(view_name)
+    try:
+        return spark.sql(
+            f"SELECT * FROM {name}(TABLE (SELECT id, value, text FROM {view_name})) ORDER BY id"  # noqa: S608
+        ).collect()
+    finally:
+        spark.catalog.dropTempView(view_name)
+
+
+def test_arrow_optimized_udf_normalizes_nested_view_types(view_type_df):
+    extract = F.udf(lambda value: value["value"], "binary", useArrow=True)
+    actual = (
+        view_type_df.filter("text IS NOT NULL")
+        .select("id", extract(nested_value()).alias("value"))
+        .orderBy("id")
+        .collect()
+    )
+    assert actual == [Row(id=id_, value=value) for id_, value, _ in VALUES[:2]]
+
+
+@pytest.mark.skipif(pyspark_version() < (4, 1), reason="arrow_udf requires PySpark 4.1+")
+@pytest.mark.parametrize("use_large", [False, True])
+def test_arrow_udf_receives_spark_boundary_types(spark, view_type_df, use_large):
+    from pyspark.sql.functions import arrow_udf
+
+    key = "spark.sql.execution.arrow.useLargeVarTypes"
+    previous = spark.conf.get(key)
+    spark.conf.set(key, str(use_large).lower())
+    try:
+        binary_type = pa.large_binary() if use_large else pa.binary()
+        string_type = pa.large_string() if use_large else pa.string()
+
+        @arrow_udf("binary")
+        def extract(value: pa.Array) -> pa.Array:
+            assert value.type.field("value").type == binary_type
+            assert value.type.field("text").type == string_type
+            assert value.type.field("values").type.value_type == binary_type
+            assert value.type.field("mapping").type.key_type == string_type
+            assert value.type.field("mapping").type.item_type == binary_type
+            return value.field("value")
+
+        actual = (
+            view_type_df.filter("text IS NOT NULL")
+            .select("id", extract(nested_value()).alias("value"))
+            .orderBy("id")
+            .collect()
+        )
+        assert actual == [Row(id=id_, value=value) for id_, value, _ in VALUES[:2]]
+    finally:
+        spark.conf.set(key, previous)
+
+
+def test_pandas_udfs_normalize_view_types(view_type_df):
+    @pandas_udf("binary")
+    def identity(value: pd.Series) -> pd.Series:
+        return value
+
+    @pandas_udf("binary", PandasUDFType.GROUPED_AGG)
+    def first(value: pd.Series):
+        return value.iloc[0]
+
+    scalar = view_type_df.select("id", identity("value").alias("value")).orderBy("id").collect()
+    aggregate = view_type_df.groupBy("id").agg(first("value").alias("value")).orderBy("id").collect()
+    window = (
+        view_type_df.select("id", first("value").over(Window.partitionBy("id")).alias("value")).orderBy("id").collect()
+    )
+    expected = [Row(id=id_, value=value) for id_, value, _ in VALUES]
+    assert scalar == aggregate == window == expected
+
+
+def test_map_udfs_normalize_view_types(view_type_df):
+    pandas_rows = view_type_df.mapInPandas(lambda batches: batches, view_type_df.schema).orderBy("id").collect()
+    assert pandas_rows == expected_rows()
+
+    if pyspark_version() >= (4,):
+
+        def identity(batches: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
+            for batch in batches:
+                assert batch.schema.field("value").type in (pa.binary(), pa.large_binary())
+                assert batch.schema.field("text").type in (pa.string(), pa.large_string())
+                yield batch
+
+        arrow_rows = view_type_df.mapInArrow(identity, view_type_df.schema).orderBy("id").collect()
+        assert arrow_rows == expected_rows()
+
+
+@pytest.mark.skipif(pyspark_version() < (4, 1), reason="UDTF table arguments require PySpark 4.1+")
+@pytest.mark.parametrize("use_arrow", [False, True])
+def test_udtf_normalizes_view_type_table_argument(spark, view_type_df, use_arrow):
+    @udtf(returnType="id int, value binary, text string", useArrow=use_arrow)
+    class Echo:
+        def eval(self, row: Row):
+            yield tuple(row)
+
+    name = f"view_type_echo_{'arrow' if use_arrow else 'row'}"
+    assert run_table_udtf(spark, view_type_df, name, Echo) == expected_rows()
+
+
+@pytest.mark.skipif(pyspark_version() < (4, 1), reason="arrow_udtf requires PySpark 4.1+")
+@pytest.mark.parametrize("use_large", [False, True])
+def test_arrow_udtf_receives_spark_boundary_types(spark, view_type_df, use_large):
+    from pyspark.sql.functions import arrow_udtf
+
+    key = "spark.sql.execution.arrow.useLargeVarTypes"
+    previous = spark.conf.get(key)
+    spark.conf.set(key, str(use_large).lower())
+    try:
+        binary_type = pa.large_binary() if use_large else pa.binary()
+        string_type = pa.large_string() if use_large else pa.string()
+
+        @arrow_udtf(returnType="id int, value binary, text string")
+        class Echo:
+            def eval(self, batch: pa.RecordBatch | pa.StructArray) -> Iterator[pa.Table]:
+                data_type = batch.type if isinstance(batch, pa.StructArray) else pa.struct(batch.schema)
+                assert data_type.field("value").type == binary_type
+                assert data_type.field("text").type == string_type
+                if isinstance(batch, pa.StructArray):
+                    table = pa.Table.from_arrays(batch.flatten(), names=[field.name for field in batch.type])
+                    for i in range(table.num_rows):
+                        yield table.slice(i, 1)
+                else:
+                    yield pa.table(batch)
+
+        name = f"view_type_arrow_echo_{'large' if use_large else 'standard'}"
+        assert run_table_udtf(spark, view_type_df, name, Echo) == expected_rows()
+    finally:
+        spark.conf.set(key, previous)
+
+
+@pytest.mark.skipif(pyspark_version() < (4, 1), reason="UDTF analyze requires PySpark 4.1+")
+def test_udtf_analyze_normalizes_view_type_table_argument(spark, view_type_df):
+    from pyspark.sql.udtf import AnalyzeArgument, AnalyzeResult
+
+    @udtf
+    class Echo:
+        @staticmethod
+        def analyze(arg: AnalyzeArgument) -> AnalyzeResult:
+            assert isinstance(arg.dataType, StructType)
+            assert isinstance(arg.dataType["value"].dataType, BinaryType)
+            assert isinstance(arg.dataType["text"].dataType, StringType)
+            return AnalyzeResult(arg.dataType)
+
+        def eval(self, row: Row):
+            yield tuple(row)
+
+    assert run_table_udtf(spark, view_type_df, "view_type_analyze_echo", Echo) == expected_rows()
diff --git a/scripts/release/list_contributors.py b/scripts/release/list_contributors.py
new file mode 100755
index 000000000..9dcb8cf54
--- /dev/null
+++ b/scripts/release/list_contributors.py
@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+"""List contributors for a commit range."""
+
+import argparse
+import json
+import shlex
+import subprocess
+import sys
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+REPOSITORY = "lakehq/sail"
+ORGANIZATION = "lakehq"
+REPOSITORY_ROOT_PATH = Path(__file__).resolve().parents[2]
+
+
+class ContributorError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class GitCommit:
+    sha: str
+    email: str
+    additions: int
+    deletions: int
+
+
+@dataclass
+class Contribution:
+    commits: int = 0
+    additions: int = 0
+    deletions: int = 0
+
+    @property
+    def lines(self) -> int:
+        return self.additions + self.deletions
+
+    def add(self, commit: GitCommit) -> None:
+        self.commits += 1
+        self.additions += commit.additions
+        self.deletions += commit.deletions
+
+
+@dataclass
+class GitHubUser:
+    login: str
+    is_bot: bool
+
+
+@dataclass
+class Contributor:
+    contribution: Contribution
+    user: GitHubUser
+    is_first_time: bool
+    is_member: bool
+
+
+def run(command: list[str]) -> str:
+    code, stdout, stderr = run_unchecked(command)
+    if code != 0:
+        message = f"`{shlex.join(command)}` failed: stdout: {stdout.strip()} stderr: {stderr.strip()}"
+        raise ContributorError(message)
+    return stdout
+
+
+def run_unchecked(command: list[str]) -> tuple[int, str, str]:
+    sys.stderr.write(f"{shlex.join(command)}\n")
+    result = subprocess.run(command, cwd=REPOSITORY_ROOT_PATH, capture_output=True, text=True, check=False)
+    return result.returncode, result.stdout, result.stderr
+
+
+def parse_json(value: str) -> Any:
+    try:
+        return json.loads(value) if value.strip() else None
+    except json.JSONDecodeError as error:
+        message = f"invalid JSON: {value}"
+        raise ContributorError(message) from error
+
+
+def parse_numstat(value: str) -> int:
+    return 0 if value == "-" else int(value)
+
+
+def collect_git_commits(commits: str) -> list[GitCommit]:
+    output = run(["git", "log", "--format=%x1e%H%x1f%ae", "--numstat", commits])
+    parsed_commits = []
+
+    for record in output.split("\x1e"):
+        lines = [line for line in record.splitlines() if line]
+        if not lines:
+            continue
+
+        try:
+            sha, email = lines[0].split("\x1f", maxsplit=1)
+        except ValueError as error:
+            message = f"could not parse git log record: {lines[0]!r}"
+            raise ContributorError(message) from error
+
+        additions = 0
+        deletions = 0
+
+        for line in lines[1:]:
+            parts = line.split("\t", maxsplit=2)
+            if len(parts) == 3:  # noqa: PLR2004
+                additions += parse_numstat(parts[0])
+                deletions += parse_numstat(parts[1])
+
+        parsed_commits.append(GitCommit(sha=sha, email=email, additions=additions, deletions=deletions))
+
+    return parsed_commits
+
+
+def first_commit(commits: str) -> str:
+    lines = run(["git", "log", "--reverse", "--format=%H", commits]).splitlines()
+    if not lines:
+        message = f"no commits found for range {commits!r}"
+        raise ContributorError(message)
+    return lines[0]
+
+
+def commit_date(commit: str) -> str:
+    return run(["git", "show", "-s", "--format=%cI", commit]).strip()
+
+
+def lookup_commit_user(commit: GitCommit) -> GitHubUser:
+    command = [
+        "gh",
+        "api",
+        f"repos/{REPOSITORY}/commits/{commit.sha}/pulls",
+        "-X",
+        "GET",
+        "-H",
+        "Accept: application/vnd.github+json",
+        "-F",
+        "per_page=1",
+    ]
+    pulls = parse_json(run(command))
+    user = pulls[0].get("user") if pulls else None
+    if user is None:
+        message = f"could not resolve a pull request author for commit {commit.sha}"
+        raise ContributorError(message)
+    return GitHubUser(
+        login=user["login"],
+        is_bot=user.get("type") == "Bot" or user["login"].endswith("[bot]"),
+    )
+
+
+def is_first_time_contributor(login: str, before: str) -> bool:
+    command = [
+        "gh",
+        "api",
+        "search/commits",
+        "-X",
+        "GET",
+        "-f",
+        f"q=repo:{REPOSITORY} author:{login} committer-date:<{before}",
+        "-F",
+        "per_page=1",
+    ]
+    data = parse_json(run(command))
+    return not data.get("total_count", 0)
+
+
+def is_organization_member(login: str) -> bool:
+    command = ["gh", "api", f"orgs/{ORGANIZATION}/members/{login}"]
+    code, stdout, stderr = run_unchecked(command)
+    data = parse_json(stdout)
+    if code == 0:
+        return True
+    if isinstance(data, dict) and data.get("status") == "404":
+        return False
+    message = f"failed to check organization membership for {login}: data: {data!r} stderr: {stderr.strip()}"
+    raise ContributorError(message)
+
+
+def collect_contributors(commits: str) -> list[Contributor]:
+    before = commit_date(first_commit(commits))
+    contributors: dict[str, Contributor] = {}
+    users_by_email: dict[str, GitHubUser] = {}
+
+    for commit in collect_git_commits(commits):
+        email = commit.email.casefold()
+        user = users_by_email.get(email)
+        if user is None:
+            user = lookup_commit_user(commit)
+            users_by_email[email] = user
+        key = user.login.casefold()
+        contributor = contributors.get(key)
+        if contributor is None:
+            # The GitHub commits API does not reliably find bot-authored commits, so skip this check for bots.
+            is_first_time = False if user.is_bot else is_first_time_contributor(user.login, before)
+            is_member = False if user.is_bot else is_organization_member(user.login)
+            contributors[key] = Contributor(
+                contribution=Contribution(),
+                user=user,
+                is_first_time=is_first_time,
+                is_member=is_member,
+            )
+            contributor = contributors[key]
+        contributor.contribution.add(commit)
+    return list(contributors.values())
+
+
+def contributor_sort_key(contributor: Contributor) -> tuple[int, int, str]:
+    return (
+        -contributor.contribution.commits,
+        -contributor.contribution.lines,
+        contributor.user.login.casefold(),
+    )
+
+
+def row(contributor: Contributor) -> str:
+    contribution = contributor.contribution
+    first_time = " (_first-time contributor_)" if contributor.is_first_time else ""
+    label = f"@{contributor.user.login}"
+    url = f"https://github.com/{contributor.user.login}"
+    commit_word = "commit" if contribution.commits == 1 else "commits"
+    return (
+        f"* ({contribution.commits} {commit_word}, "
+        f"+{contribution.additions} -{contribution.deletions}) [{label}]({url}){first_time}"
+    )
+
+
+def grouped_contributors(contributors: list[Contributor]) -> dict[str, list[Contributor]]:
+    sections: dict[str, list[Contributor]] = defaultdict(list)
+    for contributor in contributors:
+        if contributor.user.is_bot:
+            sections["Bots"].append(contributor)
+        elif contributor.is_member:
+            sections["Members"].append(contributor)
+        else:
+            sections["Community Contributors"].append(contributor)
+    return sections
+
+
+def report(contributors: list[Contributor]) -> str:
+    sections = grouped_contributors(contributors)
+    lines = []
+    for title in ["Members", "Community Contributors", "Bots"]:
+        lines.append(f"## {title}")
+        rows = [row(contributor) for contributor in sorted(sections[title], key=contributor_sort_key)]
+        lines.extend(rows or ["(none)"])
+        lines.append("")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--commits", required=True, help="commit range passed to `git log`")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        sys.stdout.write(report(collect_contributors(args.commits)))
+        sys.stdout.flush()
+    except ContributorError as error:
+        sys.stderr.write(f"error: {error}\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/scripts/release/review_packages.py b/scripts/release/review_packages.py
new file mode 100755
index 000000000..cd52007a2
--- /dev/null
+++ b/scripts/release/review_packages.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+"""Review release package artifacts and write a Markdown report."""
+
+import argparse
+import hashlib
+import sys
+import tarfile
+import zipfile
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+
+
+@dataclass(frozen=True)
+class ArchiveEntry:
+    path: PurePosixPath
+    is_dir: bool
+    mode: int | None
+    mtime: datetime | None
+    size: int
+
+
+def code_text(value: str) -> str:
+    return f"`{value}`"
+
+
+def human_size(size: int) -> str:
+    value = float(size)
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
+        if value < 1024 or unit == "TiB":  # noqa: PLR2004
+            if unit == "B":
+                return f"{size} {unit}"
+            return f"{value:.1f} {unit}"
+        value /= 1024
+    message = "unreachable"
+    raise AssertionError(message)
+
+
+def mode_text(mode: int | None) -> str:
+    return "-" if mode is None else f"{mode & 0o7777:04o}"
+
+
+def type_text(*, is_dir: bool) -> str:
+    return "Directory" if is_dir else "File"
+
+
+def mtime_text(mtime: datetime) -> str:
+    return mtime.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def digest(path: Path, algorithm: str) -> str:
+    h = hashlib.new(algorithm)
+    with path.open("rb") as file:
+        while chunk := file.read(1024 * 1024):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def directory_table(path: Path) -> list[str]:
+    rows = ["| Type | Mode | Size (B) | Modified | Name |", "| --- | ---: | ---: | ---: | --- |"]
+    for item in sorted(path.iterdir(), key=lambda value: value.name.casefold()):
+        stat = item.stat()
+        rows.append(
+            "| "
+            f"{type_text(is_dir=item.is_dir())} | "
+            f"{mode_text(stat.st_mode)} | "
+            f"{stat.st_size} | "
+            f"{code_text(mtime_text(datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)))} | "
+            f"{code_text(item.name)} |"
+        )
+    return rows
+
+
+def normalize_archive_path(value: str) -> PurePosixPath | None:
+    path = PurePosixPath(value)
+    parts = [part for part in path.parts if part not in {"", "."}]
+    if not parts or any(part == ".." for part in parts):
+        return None
+    return PurePosixPath(*parts)
+
+
+def add_parent_directories(entries: dict[PurePosixPath, ArchiveEntry], path: PurePosixPath) -> None:
+    for index in range(1, len(path.parts)):
+        parent = PurePosixPath(*path.parts[:index])
+        entries.setdefault(parent, ArchiveEntry(path=parent, is_dir=True, mode=None, mtime=None, size=0))
+
+
+def tar_gz_entries(path: Path) -> list[ArchiveEntry]:
+    entries: dict[PurePosixPath, ArchiveEntry] = {}
+    with tarfile.open(path, "r:gz") as archive:
+        for member in archive.getmembers():
+            archive_path = normalize_archive_path(member.name)
+            if archive_path is None:
+                continue
+            add_parent_directories(entries, archive_path)
+            entries[archive_path] = ArchiveEntry(
+                path=archive_path,
+                is_dir=member.isdir(),
+                mode=member.mode,
+                mtime=datetime.fromtimestamp(member.mtime, tz=timezone.utc),
+                size=member.size,
+            )
+    return sorted(entries.values(), key=lambda entry: entry.path.as_posix())
+
+
+def zip_mode(info: zipfile.ZipInfo) -> int | None:
+    mode = (info.external_attr >> 16) & 0o7777
+    return mode or None
+
+
+def zip_mtime(info: zipfile.ZipInfo) -> datetime | None:
+    try:
+        return datetime(*info.date_time, tzinfo=timezone.utc)
+    except ValueError:
+        return None
+
+
+def wheel_entries(path: Path) -> list[ArchiveEntry]:
+    entries: dict[PurePosixPath, ArchiveEntry] = {}
+    with zipfile.ZipFile(path) as archive:
+        for info in archive.infolist():
+            archive_path = normalize_archive_path(info.filename)
+            if archive_path is None:
+                continue
+            add_parent_directories(entries, archive_path)
+            entries[archive_path] = ArchiveEntry(
+                path=archive_path,
+                is_dir=info.is_dir(),
+                mode=zip_mode(info),
+                mtime=zip_mtime(info),
+                size=info.file_size,
+            )
+    return sorted(entries.values(), key=lambda entry: entry.path.as_posix())
+
+
+def archive_entries(path: Path) -> list[ArchiveEntry] | None:
+    if path.name.endswith(".tar.gz"):
+        return tar_gz_entries(path)
+    if path.suffix == ".whl":
+        return wheel_entries(path)
+    return None
+
+
+def grouped_file_name(path: PurePosixPath) -> str:
+    return f"*{path.suffix}" if path.suffix else path.name
+
+
+def entry_stats_table(entries: list[ArchiveEntry]) -> list[str]:
+    def sort_key(value: tuple[tuple[str, str], list[int]]) -> tuple[str, str]:
+        (type_, name), _ = value
+        return type_, name.casefold()
+
+    groups: dict[tuple[str, str], list[int]] = {("Directory", "*"): [entry.size for entry in entries if entry.is_dir]}
+    for entry in entries:
+        if not entry.is_dir:
+            groups.setdefault(("File", grouped_file_name(entry.path)), []).append(entry.size)
+
+    rows = [
+        "| Type | Name | Count | Maximum Size (B) | Minimum Size (B) | Total Size (B) |",
+        "| --- | --- | ---: | ---: | ---: | ---: |",
+    ]
+    for (type_, name), sizes in sorted(groups.items(), key=sort_key):
+        if sizes:
+            rows.append(f"| {type_} | {code_text(name)} | {len(sizes)} | {max(sizes)} | {min(sizes)} | {sum(sizes)} |")
+    return rows
+
+
+def top_level_content_table(entries: list[ArchiveEntry]) -> list[str]:
+    rows = ["| Type | Mode | Modified | Path |", "| --- | ---: | ---: | --- |"]
+    for entry in entries:
+        if len(entry.path.parts) > 2:  # noqa: PLR2004
+            continue
+        rows.append(
+            f"| {type_text(is_dir=entry.is_dir)} | "
+            f"{mode_text(entry.mode)} | "
+            f"{'-' if entry.mtime is None else code_text(mtime_text(entry.mtime))} | "
+            f"{code_text(entry.path.as_posix())} |"
+        )
+    if len(rows) == 2:  # noqa: PLR2004
+        rows.append("| (none) | - | - | - |")
+    return rows
+
+
+def artifact_lines(path: Path) -> list[str]:
+    entries = archive_entries(path)
+    if entries is None:
+        return []
+    size = path.stat().st_size
+    return [
+        f"### File: {code_text(path.name)}",
+        "",
+        "#### Basic Information",
+        "",
+        f"* MD5 checksum: `{digest(path, 'md5')}`",
+        f"* SHA-1 checksum: `{digest(path, 'sha1')}`",
+        f"* SHA-256 checksum: `{digest(path, 'sha256')}`",
+        f"* Size: {size} B ({human_size(size)})",
+        "",
+        "#### Top-Level Contents",
+        "",
+        *top_level_content_table(entries),
+        "",
+        "#### Entry Statistics",
+        "",
+        *entry_stats_table(entries),
+    ]
+
+
+def all_artifact_lines(path: Path) -> list[str]:
+    lines: list[str] = []
+    for artifact in sorted(
+        (item for item in path.iterdir() if item.is_file()), key=lambda value: value.name.casefold()
+    ):
+        if x := artifact_lines(artifact):
+            lines.extend(x)
+            lines.append("")
+    return lines or ["(none)"]
+
+
+def report(path: Path) -> str:
+    if not path.is_dir():
+        message = f"artifact path is not a directory: {path}"
+        raise OSError(message)
+
+    lines = [
+        "## Artifact List",
+        "",
+        *directory_table(path),
+        "",
+        "## Artifact Details",
+        "",
+        *all_artifact_lines(path),
+    ]
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--path", required=True, type=Path, help="directory containing release artifacts")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        sys.stdout.write(report(args.path))
+    except (OSError, tarfile.TarError, zipfile.BadZipFile) as error:
+        sys.stderr.write(f"error: {error}\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/scripts/release/validate_changelog.py b/scripts/release/validate_changelog.py
new file mode 100755
index 000000000..e445c86a6
--- /dev/null
+++ b/scripts/release/validate_changelog.py
@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+"""Validate release changelog entries against Git history."""
+
+import argparse
+import re
+import shlex
+import subprocess
+import sys
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+
+REPOSITORY_ROOT_PATH = Path(__file__).resolve().parents[2]
+CHANGELOG_PATH = REPOSITORY_ROOT_PATH / Path("docs/reference/changelog/index.md")
+GITHUB_PULL_URL_PATTERN = re.compile(r"^https://github\.com/lakehq/sail/pull/(\d+)(?:[/?#][^)\s]*)?$")
+GITHUB_PULL_REF_PATTERN = re.compile(r"\(#(\d+)\)$")
+MARKDOWN_HEADING_PATTERN = re.compile(r"^##\s+(.+?)\s*$")
+MARKDOWN_LINK_PATTERN = re.compile(r"\[([^\]]+)]\(([^)]+)\)")
+
+
+class ValidationError(Exception):
+    pass
+
+
+@dataclass
+class ChangelogLine:
+    line_number: int
+    text: str
+
+
+@dataclass
+class Commit:
+    title: str
+    pull_request: str | None
+
+
+@dataclass
+class ValidationData:
+    changelog_lines: list[ChangelogLine]
+    changelog_pull_requests: Counter[str]
+    commits: list[Commit]
+
+
+def find_version_section(markdown: str, version: str) -> list[ChangelogLine]:
+    lines = markdown.splitlines()
+    start = None
+
+    for index, line in enumerate(lines):
+        match = MARKDOWN_HEADING_PATTERN.match(line)
+        if match and match.group(1) == version:
+            start = index + 1
+            break
+
+    if start is None:
+        message = f"version {version!r} was not found in the changelog"
+        raise ValidationError(message)
+
+    end = len(lines)
+    for index in range(start, len(lines)):
+        if MARKDOWN_HEADING_PATTERN.match(lines[index]):
+            end = index
+            break
+
+    return [ChangelogLine(line_number=index + 1, text=lines[index]) for index in range(start, end)]
+
+
+def pull_requests_from_links(lines: list[ChangelogLine]) -> list[str]:
+    pull_requests = []
+
+    for line in lines:
+        for text, url in MARKDOWN_LINK_PATTERN.findall(line.text):
+            match = GITHUB_PULL_URL_PATTERN.match(url)
+            if match is None:
+                continue
+
+            pull_request = match.group(1)
+            expected_text = f"#{pull_request}"
+            if text != expected_text:
+                message = (
+                    f"changelog line {line.line_number}: pull request link text {text!r} does not match URL {url!r}"
+                )
+                raise ValidationError(message)
+            pull_requests.append(pull_request)
+
+    return pull_requests
+
+
+def git_log(commits: str) -> list[Commit]:
+    command = ["git", "log", "--oneline", commits]
+    result = subprocess.run(command, cwd=REPOSITORY_ROOT_PATH, capture_output=True, text=True, check=False)
+
+    if result.returncode != 0:
+        message = f"`{shlex.join(command)}` failed: stdout: {result.stdout.strip()} stderr: {result.stderr.strip()}"
+        raise ValidationError(message)
+
+    parsed_commits = []
+    for line in result.stdout.splitlines():
+        _, _, title = line.partition(" ")
+        match = GITHUB_PULL_REF_PATTERN.search(title)
+        parsed_commits.append(Commit(title=title, pull_request=match.group(1) if match else None))
+    return parsed_commits
+
+
+def append_section(lines: list[str], title: str, rows: list[str]) -> None:
+    lines.extend([title, *(rows or ["(none)"])])
+
+
+def markdown_link_text(text: str) -> str:
+    return text.replace("\\", "\\\\").replace("]", "\\]")
+
+
+def commit_titles_by_pull_request(commits: list[Commit]) -> dict[str, str]:
+    titles = {}
+    for commit in commits:
+        if commit.pull_request is not None:
+            titles.setdefault(commit.pull_request, commit.title)
+    return titles
+
+
+def annotated_changelog(lines: list[ChangelogLine], commits: list[Commit]) -> list[str]:
+    titles = commit_titles_by_pull_request(commits)
+
+    def replace_link(match: re.Match[str]) -> str:
+        _, url = match.groups()
+        url_match = GITHUB_PULL_URL_PATTERN.match(url)
+        if url_match is None:
+            return match.group(0)
+
+        title = titles.get(url_match.group(1))
+        if title is None:
+            return match.group(0)
+        return f"[{markdown_link_text(title)}]({url})"
+
+    return [MARKDOWN_LINK_PATTERN.sub(replace_link, line.text) for line in lines]
+
+
+def report(data: ValidationData) -> str:
+    changelog_pull_request_set = set(data.changelog_pull_requests)
+    known_pull_requests = {commit.pull_request for commit in data.commits if commit.pull_request is not None}
+
+    known_rows = []
+    for commit in data.commits:
+        pull_request = commit.pull_request
+        if pull_request is not None and pull_request in changelog_pull_request_set:
+            count = data.changelog_pull_requests[pull_request]
+            known_rows.append(f"* ({count}) {commit.title}")
+
+    missing_rows = [
+        f"* (0) {commit.title}" for commit in data.commits if commit.pull_request not in changelog_pull_request_set
+    ]
+
+    unknown_rows = [
+        f"* ({data.changelog_pull_requests[pull_request]}) ??? (#{pull_request})"
+        for pull_request in sorted(changelog_pull_request_set - known_pull_requests, key=int)
+    ]
+
+    lines = []
+    append_section(lines, "## Known Commits in the Changelog", known_rows)
+    lines.append("")
+    append_section(lines, "## Known Commits Not in the Changelog", missing_rows)
+    lines.append("")
+    append_section(lines, "## Unknown Commits in the Changelog", unknown_rows)
+    lines.append("")
+    append_section(lines, "## Annotated Changelog", annotated_changelog(data.changelog_lines, data.commits))
+    return "\n".join(lines) + "\n"
+
+
+def has_unknown_commits(data: ValidationData) -> bool:
+    changelog_pull_request_set = set(data.changelog_pull_requests)
+    known_pull_requests = {commit.pull_request for commit in data.commits if commit.pull_request is not None}
+    unknown = changelog_pull_request_set - known_pull_requests
+    return bool(unknown)
+
+
+def load_validation_data(version: str, commits: str) -> ValidationData:
+    markdown = CHANGELOG_PATH.read_text(encoding="utf-8")
+    changelog_lines = find_version_section(markdown, version)
+    return ValidationData(
+        changelog_lines=changelog_lines,
+        changelog_pull_requests=Counter(pull_requests_from_links(changelog_lines)),
+        commits=git_log(commits),
+    )
+
+
+def validate(version: str, commits: str) -> None:
+    data = load_validation_data(version, commits)
+    output = report(data)
+    sys.stdout.write(output)
+    sys.stdout.flush()
+
+    if has_unknown_commits(data):
+        message = "found unknown commits in the changelog"
+        raise ValidationError(message)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--version", required=True, help="version heading to validate")
+    parser.add_argument("--commits", required=True, help="commit range passed to `git log`")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        validate(args.version, args.commits)
+    except ValidationError as error:
+        sys.stderr.write(f"error: {error}\n")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/scripts/spark-config/README.md b/scripts/spark-config/README.md
deleted file mode 100644
index e671af2a5..000000000
--- a/scripts/spark-config/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# Spark Configuration Data Collection
-
-This is not part of the daily developer workflow.
-
-This directory contains the script to collect Spark SQL configuration information
-into a data file.
-The data file is used to generate the Rust code for the Spark configuration.
-
-Run the following command **in the project root directory** to update the data file.
-Please commit the changes if any.
-
-```bash
-env SPARK_LOCAL_IP=127.0.0.1 \
-  hatch run scripts/spark-config/generate.py \
-  -o crates/sail-spark-connect/data/spark_config.json
-```
diff --git a/scripts/spark-config/generate.py b/scripts/spark-config/generate.py
index c434554b8..3f369be69 100755
--- a/scripts/spark-config/generate.py
+++ b/scripts/spark-config/generate.py
@@ -2,6 +2,7 @@
 import argparse
 import contextlib
 import json
+import os
 
 import pyspark
 from py4j.protocol import Py4JError
@@ -57,6 +58,11 @@ def collect_spark_config(spark):
     clazz = spark._jvm.java.lang.Class.forName("org.apache.spark.sql.internal.SQLConf$")
     obj = clazz.getDeclaredField("MODULE$").get(None)
 
+    # Initialize the Scala `Connect$` object so its options are registered in `SQLConf`.
+    # In Spark 3.x this class comes from the `spark-connect` package, which is
+    # available through Spark's class loader rather than the JVM's default class loader.
+    spark._jvm.org.apache.spark.util.Utils.classForName("org.apache.spark.sql.connect.config.Connect$", True, False)
+
     deprecation_map = {
         x.key(): {"version": x.version(), "comment": x.comment()}
         for x in to_java_map(obj.deprecatedSQLConfigs()).values()
@@ -87,6 +93,14 @@ def main():
     parser.add_argument("-o", "--output", required=True)
     args = parser.parse_args()
 
+    # PySpark 3.x does not load the Spark Connect Scala package by default.
+    # When collecting the Spark 3.x configuration, we set `PYSPARK_SUBMIT_ARGS`
+    # so that package is available and its options can be registered.
+    if pyspark.__version__.startswith("3."):
+        os.environ["PYSPARK_SUBMIT_ARGS"] = (
+            f"--packages org.apache.spark:spark-connect_2.12:{pyspark.__version__} pyspark-shell"
+        )
+
     spark = pyspark.sql.SparkSession.builder.master("local[1]").appName("Test").getOrCreate()
 
     with open(args.output, "w") as f:
diff --git a/scripts/spark-gold-data/bootstrap.sh b/scripts/spark-gold-data/bootstrap.sh
index 17bffdeb3..ff4935640 100755
--- a/scripts/spark-gold-data/bootstrap.sh
+++ b/scripts/spark-gold-data/bootstrap.sh
@@ -7,6 +7,8 @@ if [ -z "${JAVA_HOME:-}" ]; then
   exit 1
 fi
 
+spark_version="4.2.0"
+
 project_path="$(git rev-parse --show-toplevel)"
 
 scripts_path="${project_path}/scripts/spark-gold-data"
@@ -15,7 +17,7 @@ output_path="${project_path}/crates/sail-spark-connect/tests/gold_data"
 
 source "${project_path}/scripts/shell-tools/git-patch.sh"
 
-apply_git_patch "${project_path}"/opt/spark "v4.1.1" "${scripts_path}/spark-4.1.1.patch"
+apply_git_patch "${project_path}"/opt/spark "v${spark_version}" "${scripts_path}/spark-${spark_version}.patch"
 
 cd "${project_path}"/opt/spark
 
@@ -38,14 +40,6 @@ echo "Removing existing test data..."
 rm -rf "${output_path}"
 mkdir -p "${output_path}"
 
-cat < "${output_path}/README.md"
-# Gold Data for Spark Tests
-
-This directory contains the gold data for the Spark tests.
-All the files in this directory, including this README file, are auto-generated.
-Please do not modify them manually.
-EOF
-
 cargo run -p sail-gold-test --bin spark-gold-data -- \
   --input "${logs_path}" \
   --output "${output_path}"
diff --git a/scripts/spark-gold-data/spark-4.1.1.patch b/scripts/spark-gold-data/spark-4.2.0.patch
similarity index 90%
rename from scripts/spark-gold-data/spark-4.1.1.patch
rename to scripts/spark-gold-data/spark-4.2.0.patch
index 04de941b1..95ca8abab 100644
--- a/scripts/spark-gold-data/spark-4.1.1.patch
+++ b/scripts/spark-gold-data/spark-4.2.0.patch
@@ -1,8 +1,8 @@
 diff --git a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
-index 810bdabebb3..079df78998d 100644
+index 3fac57dbe5d..38584fb1b8c 100644
 --- a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
 +++ b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
-@@ -408,6 +408,8 @@ private[spark] object Logging {
+@@ -413,6 +413,8 @@ private[spark] object Logging {
    @volatile private[spark] var sparkShellThresholdLevel: Level = null
    @volatile private[spark] var setLogLevelPrinted: Boolean = false
  
@@ -11,13 +11,13 @@ index 810bdabebb3..079df78998d 100644
    val initLock = new Object()
    try {
      // We use reflection here to handle the case where users remove the
-diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
-index 3b0ec30ee86..bdb6c5e200b 100644
---- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
-+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
-@@ -89,15 +89,54 @@ abstract class SparkFunSuite
- 
-   protected val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
+diff --git a/core/src/test/scala/org/apache/spark/SparkTestSuite.scala b/core/src/test/scala/org/apache/spark/SparkTestSuite.scala
+index 10504684be9..f3bb4a8f15e 100644
+--- a/core/src/test/scala/org/apache/spark/SparkTestSuite.scala
++++ b/core/src/test/scala/org/apache/spark/SparkTestSuite.scala
+@@ -91,15 +91,54 @@ trait SparkTestSuite
+   protected def regenerateGoldenFiles: Boolean =
+     System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
  
 +  /**
 +   * The environment variable for specifying the directory to write test suite data.
@@ -34,7 +34,7 @@ index 3b0ec30ee86..bdb6c5e200b 100644
 +      )
 +    }
 +    val className = this.getClass.getName.stripSuffix("$").split('.').last
-+    new java.io.File(outputDir, s"$className.jsonl")
++    new File(outputDir, s"$className.jsonl")
 +  }
 +
 +  /** Write test data to the output file. */
@@ -71,7 +71,7 @@ index 3b0ec30ee86..bdb6c5e200b 100644
        // Avoid leaking map entries in tests that use accumulators without SparkContext
        AccumulatorContext.clear()
 diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
-index 32270df0a98..a52bb7920f6 100644
+index c997a102a70..93403dc7454 100644
 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
 +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
 @@ -35,13 +35,25 @@ import org.apache.spark.sql.errors.QueryParsingErrors
@@ -111,10 +111,10 @@ index 32270df0a98..a52bb7920f6 100644
    }
  
 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala
-index 216136d8a7c..35d9aa9c7c4 100644
+index 29bf924f244..dee33a596e2 100644
 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala
 +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala
-@@ -27,6 +27,7 @@ import org.apache.spark.sql.errors.QueryParsingErrors
+@@ -28,6 +28,7 @@ import org.apache.spark.sql.errors.QueryParsingErrors
  import org.apache.spark.sql.internal.SQLConf
  import org.apache.spark.sql.types.StructType
  
@@ -122,7 +122,7 @@ index 216136d8a7c..35d9aa9c7c4 100644
  /**
   * Base class for all ANTLR4 [[ParserInterface]] implementations.
   */
-@@ -35,7 +36,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
+@@ -36,7 +37,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
  
    /** Creates Expression for a given SQL string. */
    override def parseExpression(sqlText: String): Expression =
@@ -131,7 +131,7 @@ index 216136d8a7c..35d9aa9c7c4 100644
        val ctx = parser.singleExpression()
        withErrorHandling(ctx, Some(sqlText)) {
          astBuilder.visitSingleExpression(ctx)
-@@ -44,7 +45,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
+@@ -45,7 +46,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
  
    /** Creates TableIdentifier for a given SQL string. */
    override def parseTableIdentifier(sqlText: String): TableIdentifier =
@@ -140,7 +140,7 @@ index 216136d8a7c..35d9aa9c7c4 100644
        val ctx = parser.singleTableIdentifier()
        withErrorHandling(ctx, Some(sqlText)) {
          astBuilder.visitSingleTableIdentifier(ctx)
-@@ -53,7 +54,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
+@@ -54,7 +55,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
  
    /** Creates FunctionIdentifier for a given SQL string. */
    override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = {
@@ -149,7 +149,7 @@ index 216136d8a7c..35d9aa9c7c4 100644
        val ctx = parser.singleFunctionIdentifier()
        withErrorHandling(ctx, Some(sqlText)) {
          astBuilder.visitSingleFunctionIdentifier(ctx)
-@@ -63,7 +64,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
+@@ -64,7 +65,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
  
    /** Creates a multi-part identifier for a given SQL string */
    override def parseMultipartIdentifier(sqlText: String): Seq[String] = {
@@ -158,7 +158,7 @@ index 216136d8a7c..35d9aa9c7c4 100644
        val ctx = parser.singleMultipartIdentifier()
        withErrorHandling(ctx, Some(sqlText)) {
          astBuilder.visitSingleMultipartIdentifier(ctx)
-@@ -73,7 +74,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
+@@ -74,7 +75,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
  
    /** Creates LogicalPlan for a given SQL string of query. */
    override def parseQuery(sqlText: String): LogicalPlan =
@@ -167,7 +167,7 @@ index 216136d8a7c..35d9aa9c7c4 100644
        if (!SQLConf.get.getConf(SQLConf.LEGACY_PARSE_QUERY_WITHOUT_EOF)) {
          val ctx = parser.singleQuery()
  
-@@ -90,7 +91,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
+@@ -91,7 +92,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface {
      }
  
    /** Creates LogicalPlan for a given SQL string. */
diff --git a/scripts/spark-tests/plugins/spark.py b/scripts/spark-tests/plugins/spark.py
index 00cc88c97..b2866cc29 100644
--- a/scripts/spark-tests/plugins/spark.py
+++ b/scripts/spark-tests/plugins/spark.py
@@ -340,6 +340,10 @@ def _spark_major_version() -> int:
         keywords=["pyspark.sql.functions.java_method"],
         reason="JVM-dependent test",
     ),
+    TestMarker(
+        keywords=["pyspark.sql.functions.reflect"],
+        reason="JVM-dependent test",
+    ),
     # We skip all the streaming tests since some of them are slow,
     # and some of them test behaviors that are tied to the specific JVM implementation
     # of Spark Structured Streaming.
@@ -419,6 +423,14 @@ def _spark_major_version() -> int:
         reason="Spark 3.x doctest uses the JVM-dependent RDD API",
         spark_major_version_less_than=4,
     ),
+    TestMarker(
+        keywords=["test_parity_readwriter.py", "test_create_without_provider"],
+        reason=(
+            "Spark 3 requires Hive when no provider is specified, while Spark 4 uses the default provider; "
+            "Sail follows Spark 4"
+        ),
+        spark_major_version_less_than=4,
+    ),
 ]
 
 
diff --git a/scripts/spark-tests/spark-3.5.7.patch b/scripts/spark-tests/spark-3.5.9.patch
similarity index 98%
rename from scripts/spark-tests/spark-3.5.7.patch
rename to scripts/spark-tests/spark-3.5.9.patch
index f42e08dbe..6702ef31d 100644
--- a/scripts/spark-tests/spark-3.5.7.patch
+++ b/scripts/spark-tests/spark-3.5.9.patch
@@ -1,8 +1,8 @@
 diff --git a/pom.xml b/pom.xml
-index a0e25ce4d8d..261a2b03380 100644
+index 49eca0f7555..98bac4fb95c 100644
 --- a/pom.xml
 +++ b/pom.xml
-@@ -3357,6 +3357,10 @@
+@@ -3353,6 +3353,10 @@
              
                com.google.common
                ${spark.shade.packageName}.guava
@@ -132,7 +132,7 @@ index a063f27c9ea..784161bbade 100644
  
          cls.connect.set_hook("readTable", cls._read_table)
 diff --git a/python/pyspark/testing/objects.py b/python/pyspark/testing/objects.py
-index 5b97664afbd..b9deeef91c8 100644
+index 2acbd0ba1b4..3232d6c2478 100644
 --- a/python/pyspark/testing/objects.py
 +++ b/python/pyspark/testing/objects.py
 @@ -45,7 +45,7 @@ class ExamplePointUDT(UserDefinedType):
@@ -145,7 +145,7 @@ index 5b97664afbd..b9deeef91c8 100644
      @classmethod
      def scalaUDT(cls):
 diff --git a/python/setup.py b/python/setup.py
-index ddd961d0412..f58d965e737 100755
+index 16ae358caec..f1dff5d4afd 100755
 --- a/python/setup.py
 +++ b/python/setup.py
 @@ -183,6 +183,9 @@ try:
diff --git a/scripts/spark-tests/spark-4.1.1.patch b/scripts/spark-tests/spark-4.2.0.patch
similarity index 91%
rename from scripts/spark-tests/spark-4.1.1.patch
rename to scripts/spark-tests/spark-4.2.0.patch
index 28e550b64..ae1e1dae3 100644
--- a/scripts/spark-tests/spark-4.1.1.patch
+++ b/scripts/spark-tests/spark-4.2.0.patch
@@ -1,8 +1,8 @@
 diff --git a/python/packaging/classic/setup.py b/python/packaging/classic/setup.py
-index 54ec4abe3be..28edac8f441 100755
+index 122c4ca6d4b..8a17c9d3878 100755
 --- a/python/packaging/classic/setup.py
 +++ b/python/packaging/classic/setup.py
-@@ -206,6 +206,9 @@ try:
+@@ -214,6 +214,9 @@ try:
      copyfile("pyspark/shell.py", "pyspark/python/pyspark/shell.py")
  
      if in_spark:
@@ -12,7 +12,7 @@ index 54ec4abe3be..28edac8f441 100755
          # !!HACK ALTERT!!
          # `setup.py` has to be located with the same directory with the package.
          # Therefore, we copy the current file, and place it at `spark/python` directory.
-@@ -308,6 +311,7 @@ try:
+@@ -316,6 +319,7 @@ try:
              "pyspark.pandas.typedef",
              "pyspark.pandas.usage_logging",
              "pyspark.pipelines",
@@ -20,7 +20,7 @@ index 54ec4abe3be..28edac8f441 100755
              "pyspark.python.pyspark",
              "pyspark.python.lib",
              "pyspark.testing",
-@@ -318,6 +322,62 @@ try:
+@@ -326,6 +330,64 @@ try:
              "pyspark.errors.exceptions",
              "pyspark.examples.src.main.python",
              "pyspark.logger",
@@ -64,6 +64,7 @@ index 54ec4abe3be..28edac8f441 100755
 +            "pyspark.resource.tests",
 +            "pyspark.sql.tests",
 +            "pyspark.sql.tests.arrow",
++            "pyspark.sql.tests.coercion",
 +            "pyspark.sql.tests.connect",
 +            "pyspark.sql.tests.connect.arrow",
 +            "pyspark.sql.tests.connect.client",
@@ -76,14 +77,15 @@ index 54ec4abe3be..28edac8f441 100755
 +            "pyspark.sql.tests.pandas.streaming",
 +            "pyspark.sql.tests.plot",
 +            "pyspark.sql.tests.streaming",
-+            "pyspark.sql.tests.udf_type_tests",
 +            "pyspark.streaming.tests",
 +            "pyspark.testing.tests",
 +            "pyspark.tests",
++            "pyspark.tests.upstream",
++            "pyspark.tests.upstream.pyarrow",
          ],
          include_package_data=True,
          package_dir={
-@@ -338,10 +398,29 @@ try:
+@@ -346,10 +408,29 @@ try:
                  "start-history-server.sh",
                  "stop-history-server.sh",
              ],
@@ -106,7 +108,7 @@ index 54ec4abe3be..28edac8f441 100755
              "pyspark.python.lib": ["*.zip"],
 -            "pyspark.data": ["*.txt", "*.data"],
 +            "pyspark.data": ["*.txt", "*.data", "artifact-tests/*.jar"],
-             "pyspark.licenses": ["*.txt"],
+             "pyspark.licenses": ["*"],
              "pyspark.examples.src.main.python": ["*.py", "*/*.py"],
 +            "pyspark.ml.tests": ["typing/*.yml"],
 +            "pyspark.sql.tests": ["typing/*.yml"],
@@ -115,10 +117,10 @@ index 54ec4abe3be..28edac8f441 100755
          scripts=scripts,
          license="Apache-2.0",
 diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py
-index 80d83c69c45..2625eae10ad 100644
+index b2e30a8325d..b99eed69504 100644
 --- a/python/pyspark/sql/connect/client/core.py
 +++ b/python/pyspark/sql/connect/client/core.py
-@@ -344,6 +344,10 @@ class DefaultChannelBuilder(ChannelBuilder):
+@@ -352,6 +352,10 @@ class DefaultChannelBuilder(ChannelBuilder):
              session = PySparkSession._instantiatedSession
  
              if session is not None:
@@ -129,17 +131,8 @@ index 80d83c69c45..2625eae10ad 100644
                  jvm = PySparkSession._instantiatedSession._jvm  # type: ignore[union-attr]
                  return getattr(
                      getattr(
-@@ -1249,6 +1253,8 @@ class SparkConnectClient(object):
-         """
-         Close the channel.
-         """
-+        if self._closed:
-+            return
-         ExecutePlanResponseReattachableIterator.shutdown()
-         self._channel.close()
-         self._closed = True
 diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
-index b789d7919c9..a4ec3bc4bf7 100755
+index 516c3ad3751..72aac8bfe73 100755
 --- a/python/pyspark/sql/tests/connect/test_connect_basic.py
 +++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
 @@ -82,6 +82,11 @@ class SparkConnectSQLTestCase(ReusedMixedTestCase, PandasOnSparkTestUtils):
@@ -165,7 +158,7 @@ index b789d7919c9..a4ec3bc4bf7 100755
  
      @classmethod
 diff --git a/python/pyspark/testing/objects.py b/python/pyspark/testing/objects.py
-index 5b97664afbd..b9deeef91c8 100644
+index 2acbd0ba1b4..3232d6c2478 100644
 --- a/python/pyspark/testing/objects.py
 +++ b/python/pyspark/testing/objects.py
 @@ -45,7 +45,7 @@ class ExamplePointUDT(UserDefinedType):